diff --git a/.gitattributes b/.gitattributes index 865c65f9894a4657d6bbe37191e2de32f4c61661..9ae8b32d4161f6188337f4118a86e9ae01fcba45 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1692,3 +1692,204 @@ hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_REVERSE_CFG/OmniAvat hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_REVERSE_CFG/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward_reverse_cfg/wandb/run-20260502_023741-vye7r2ja/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_REVERSE_CFG/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward_reverse_cfg/wandb/run-20260502_023741-vye7r2ja/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_REVERSE_CFG/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward_reverse_cfg/wandb/run-20260502_023741-vye7r2ja/run-vye7r2ja.wandb filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_100_02a9469e6a8f62c40ad7.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_200_2af0f38fdf389dae8d28.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_300_38c363a3e652a07dc1bb.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_400_16d8fbd2f6c73ac8872f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_500_7abed1a68789457ad3a0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_1fc3b5e201154e6e34af.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_24922a8666c073275c9a.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_4fcbdc3dc61e38daa972.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_62bed84a4aa382a8a841.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_6986716832066cc0b4b3.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_989042d0d296c8757b9c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_a311ef087e615abc4b88.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_a9ee1f7ea80e20a77ba8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_cd952d64b92f2ae55e8d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_e7625fd6dbecfcd47200.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_13b0ba1edef33da14281.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_23962546ea0008325d27.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_44858b0cbf80e0776333.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_514ead401e4d4d48563b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_76d85fc2939323206846.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_7d946e28fdb7615e3a75.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_94dc14699d4b02dcf3e4.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_9527a798369191a957ee.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_a016c316831a3184c893.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_d398a1b31d3dd0daddaf.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_104a15bb2c320b7cd460.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_2d21142e2c65dd101874.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_89aa3c5736b12b29c461.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_8ca35e1e0f4c87746fcb.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_989687e266022d183500.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_9f19fcc77f53bceef388.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_ca18021f1758bd6e5e2f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_d348cf69473e92cd8bb9.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_e61d818001c341ef6fcd.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_f77db77f9edd6deeea55.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_02fc4d5308d7410b9769.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_0af5a053a398519cd953.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_12c1b7ac5a908057071a.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_15143e2dc19672250f8c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_1f229e284511cd47616d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_8fd7e644dfc8020f3337.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_95156dbf3f0b2df3db0c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_a8cd1b085302aefed74e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_b993b55b7abeeda934f0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_d7dbecce91d9e83ae5e1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_00ffc87bb9b6e918112e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_3d30c382480ad9b7821c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_3d83faf19efb58bd1ca3.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_77b86d717bc4e6ae54de.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_921361a9f8134e388fa7.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_92cff2dca642f0ba9d81.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_9e98e7916427159cdbe5.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_cb02a559e7336c93031b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_db8c7492f671ca319ef5.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_f88305a649c54fbcf67c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_16b78933092d6668cdb7.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_46424061a7c159bc5d15.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_56348bb82660a9b2298b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_57748de0546e2eca3715.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_5fad37a16041bbf6fa85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_8cadb1fdd841ccef6984.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_9dd369241b0d624306b6.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_d4cae118f95943b9957a.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_dfdfdd14726cd8f9b0e4.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_e3853e4d45d36ffc859e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/run-pdowrk23.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..0e6deed4dbd0b78756cf0defcdb0671fa096cd66 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e94c3f7598c50b0ce4d2bc5f0e2c799abf3544f642f1c2915181703c60995b99 +size 635553 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..20bc67e60636252568680ab0839e363c9462f23d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b294055b3c11d504a03f2c99315d6c9d2ead53461f646c5fb98e20e4f108484 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3ccc4b0356aa9ad578ff51d0e449ac9f8809a2ba --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d583b4611d72cba98adc365a287151cec148bfe1103234416c0e1e0811dba84 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..702c4f93174ed22f242e7d480bf065764a9f83dc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbec14a25666e3197cac6799c5c98f4bfd92fb062e609953e8c7bdf12368cfd1 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1e6e446b1b90d4442e13f32139fb0e7f747ed9d2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:693760f7cc889baae475704b43f686f72ee51e2f582c5e09a675d3f89277dff7 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f6f5d975cd484b820d1476ecb0443a911eefd6c3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c359b9a91c49b1d11fe9a53d4ec157a40405f274c1a29d804d0eed32779062a0 +size 635546 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..dee8cbedc7fd3d55c57d736f8286d8f26c081e69 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd91d24e01bd47ce8d6ac9198b467506d44e681aeedaca675d4a52b508089ce4 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e101ecd198208fa14dce1a1ae1921f63c5bc191c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:031d1630debbb6cf5cbff6e78d0a963b0c1cc5830e4cd20f617d03a97b03e51a +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e75ef1b31a677f833a70e4017fb96df582fa80ba --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8641cf80c96c11a94833a063313febb27c98bc2ef361419994d24828753afb5a +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..860f7ed043184296757b73b54c01b3f54d68a791 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5ba742e0888eed8011c81203a512f315f965fa4b69215e39c527c78c18bd598 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f637e0f6792d8ae1960e0e448573321eacbddc4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9e7bf7b483afea9b5f5fc9f6d1a8368eea2246e5e5404c08b9fba8acdc0064 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c79b7d5cb6798a930d28a00f7ce5d84df3063127 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8988422d4b017eeb5a38827c87cec6c2f3a68f00750f86d211ee541ce037545 +size 635553 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3406da4aae24ff8b5778e357469c812a67e5b6de --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c31507806fead5d9950b18e9d9933bc909e9e96bd689992dac202cad5b96139b +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..638878345710c36e7b27dfe87656dd0b32b3ba58 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ef64b81e39f475d95b30b3c75c5ba3dd73bd4ae7e11e4079b9340cc3e2ce89a +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a1ca53707363561d246c0fe29459ba4103e9bcfa --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97535cb74f5aee7b6fdc7e6907d61ad531245d6e59dc5bfdd591d3e257fd3233 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2a1a4591c5e19bd9ebd47dfd2eaeac0e3b24a564 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53e4878a02c4a28f75a74ef19b0b31edafeb17443ea00fa7d77c28a68c9e81f0 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d4c9365bca0e74dc066d4bc7f4a5a841ca092fef --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df678a957d3d8f999ea0266d416a1dc7b30f3bab253cbd79fda24e4d5f0e6e3f +size 635546 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..90df10bb2347a7aff159d83ac620450e9c226090 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c38956a63aa02b3ae9139d834d55e6e24620426f482cbd49f5bfdd4fb83aa98 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1c83802a6327aa94bd9c5a90dde80a9d960eb09b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cc12154daaa2b480d8083840255728ba1a889a7b46d842971c3a11068b39b40 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ea452bb85bd6cfee1d1470ef8f5f6613aa9f9373 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26157a6dce9cd2fb2fbf5d4607c3db597b6f0c48ea983ce30431e71b7497688d +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..87aa0f8c588ca31ab396ca0e07f8b41388f78d20 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e94fcbb071df3cecc0ab89390260926f9e49b7b81d94a570e68463c38868a65 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.pth new file mode 100644 index 0000000000000000000000000000000000000000..94689c6d88777305ec14be3e82dcf676c28c81a1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d49c29dddf4cff9abe84c331dfad17799824bf4158c145f77812c5430228f1 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..b0d61b5b3aac81a503a9e0bff54a8d019bcc7480 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97dcb5d5cc090af22a66da29eab5a734c1a874e6a712db981fd814ea324bf556 +size 635553 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d1e100a59d6a3d03c8ccc7c9d489b209d41a4df9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e9f85ab5a074786dc005fa59b32393d9801fa23c11aa32c94a47077d8399353 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..68c737bc0283ab2a017f89f3a7f1e8e2ed15f3ee --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd889270cdf93f05f4269bfc614b8edabcab8db92fa0dfebf97a17aaa4171bf3 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..692f065d1a374b80f50986c0336d0cfa6d928451 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4d2c03893c326f53ea21417e7d6259ad30773be84ce39bb7800de643d9076b4 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b0802250e71d3d67afa04e8e1311bf27a0238ffb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5157a6bc6e8457eec67ef0688977beefcf560bc2283fed884f9afa3c79704712 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..014813e99d6d43e6427432a5dfc600a9e8edd5c0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38208f53e213e82119b48615264b24d7cbdeda4a6ad7450c7cc2668c583927fb +size 635546 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..31a2e36ca105d4587ce889b9abb2384ce83ff48b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c32f41873ad614b496345bac4910374fde4e78ebfbb2f118eb119b2d7ba57d52 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9aadd8af4b33447aab17e22bd65ec9f34c2a5382 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d35f3a809abe85afc72d1d814bcc35e7d62cc6793f20109f94ce2c7a802244c0 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..39c7008424d16bb337586b9165dade017a33ff5a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c69afbfde5789b9d7756e1fc850eab0ec969b5f6e93089bb26c355212a6965b1 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..154913a305f030f62dd839123ed7799db0a739c3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efebfc0faf28d6b1a14315ad2ce9c6ed90aabe34445634aff5b321d2d3edbdd9 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.pth new file mode 100644 index 0000000000000000000000000000000000000000..36cdfef63708c11e993e5164fd9859eb64d6d33e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463104a57d6bfde142e5cd01a059afdb8b99dad40c73e04a236502ee34fb4d66 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..92157db014370dfe1f19fffc13980c0ac8735aae --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce2952034e564b56d11bd701d2ee551e586c728086b34e65649be6b6fc620999 +size 635553 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..eadf0fea7f7919b030ee7de9825394756714ef68 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0919e7fa3ea06be4f32b367bdddd2b1cc9146d70bc3936a3dfe55f298de355ce +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c19c4340aa8188cefbba7244c0aa9070cdbfc266 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97672d87903e1c8c33ea8bd9aa63eb8e727f6fad133ebe135354b7baab832c4 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..413e2508ead2d2717dccc91b2fcad6d368a59dc6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64e9d9897c19f020291f489868c5a2e71bad616a5caafb58aa6b83886ff8bfac +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2cf35bc6540f41c272e16a44dbb0c9263a179800 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a840340abe125cdee12f57e5e9b04d4a8b275fcedba3cc93936b0354c405f667 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..57d062ce210b7fd36eb6cf3954261b73a4e71172 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7ed890246de18ca56f31d3f46c87713843687eb9d8d71880e361d60d31af614 +size 635546 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0ac4836ec528434bda16521b52b58d9297f8adbd --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c2ef2c625a769732d5fd656f4bf5a807b95c8964deb06d5134998763a6f0bf3 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bef9abfc23faeb6d1d3a4ae2dade61b0d18a3f21 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12f05dc4eeeae11e333390506d651f6b33bce7ce0a30de7732eab6aab2dc06e8 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..902f0e9d0e9dac3e6c7fe262e5f14e07dec215cd --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a45f6770c535078697b582df70c819a66d2f7cd32c5bbf463fb0318fae1128ff +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f1de05bf7fb4d281049483e83c92d87dc91f99a8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ec19da192e13dd4cb2f8fd65ce652127e0b76d1a2c68b7dc777358eba54b24c +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.pth new file mode 100644 index 0000000000000000000000000000000000000000..0deb1499fd51197d9815f644a93381fbe0678161 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31545bbbd2979dddf082789dbf999d9324dc9febf1b016304ddc713ede429c3e +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6adb707d9fa4f948e9f0f4bd116c3a9ec45ed92c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7928685de559ecde3cf7a736967c1a8d7fd8addcd05b349d516cebdc47e3fd2c +size 635553 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..517b5b9e0936c03c894e1ce6e817ec9a357bfc34 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25d4aecd5ed07890037064835b154c12a7749409d385b9a793a2ec8b187145e7 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..97ea41adb57181a284ca1dd8c079c5715acaf41f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89b9d2fc01543c09161afe04a3bd66576edf3fa27bd717400b42bb2528fcc02d +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b030e148795995988d681c2d566e591cae33f5cc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d60de797a5f952552eec2e8d4bdc52f7abf87e588093fa76a07aade3def8e9ca +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..af9c486e00b2639f1bd65b79909d57c8750a1304 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55741ffc47f34d5369af802bceaa051d6f167b25a93683ad74bfe2d8f532921b +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..3c54520f1554a9abc6982a1c24505baa5a48f1ab --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c35a24c26eed62e54656199aa52e450a7496e62eee6c5e1f7ad323b0a88407e4 +size 635546 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..cbd7645d5020e5404f33aa39e707c77b807542f1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59c194eac0ed77d3fbc449dac65b202e1af0c6191ef0b4b88688d18dfa3b6a08 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..72ddbb6781eee66ae1c240ded7ddb45d924301d8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b49ab382c87bbc1d7344c92d5cc1c24c62df24d14cbdc7136c84f43de8be0c9f +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b0a039438355fd173a8f2553a3d03d13bcdbdbd2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:348b25f0b1b1d9caec014f6527293cab7c45dee988a05ff924ad7679b05d5325 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b162a83c454cb1add2085a5cac34cd6824c26575 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81701b049e55f4fe60785c90ee498aa54f60646f794f570c38270b37b82acf62 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..df0fb483027e40ac33d6a36f4b38fcfd244d6a44 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2110994f1f90ec15f3f831251e70d29fe1b83f8a35bb969069c4ac3f211c6d +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..62a8cf3c4b1e0ac8aa75df0ee7aabdf8467743c0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a4f615d1a9f02a4165e9de2c72d7e9afdb5c3bad6a31c6c2a6eedd206c82d6a +size 635553 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..16448626029e0500dd9d11ebf68b87e193b19258 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec3355bb7286f3f69fc283e577dbc64564b54dd7492d41c1586c6b189559ed34 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..5b18dacaeb9142aaa4ad3e8ea6f3e34835e799bb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2404d35878a2849760955390270d40aa63ec8f2f90c678046f8af063e7682332 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7a11919da5b725f9cb3fac45325a1bd521d571ec --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:656ea257c3d06850d7730aedaa6b872f6adf5312d0ce204ee77a9fa6b28786b0 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..23d26d99e8818fdc0110aa96dfedda849edcc06e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a752920c18fbde047359f69eb5a2f6a5b5defaa48c51d61055c05479f0d12375 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d206e69b5e2292bed602720dfa2100225e41af9f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a7ef17afb7a7d9757f7a5daf957a781bfa1ba2cd39e5f4345cc65bbdd6d9b86 +size 1722917 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8a9ba312c1fc2682981eb8280b16dc256d1e6bd8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9973dc8407277b407552d5c31542d44c781fda1a5d3d1aa399529183d2236d +size 2847149355 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..48821c72d8c21755ce3a1312271d8eb25005b5d4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16ab6959c122aeed5a1b2b38a34f211a9fef54e5a7ec1e5018fedd1ac5f6a8ce +size 2845267132 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..970fb754a9dc77ca95f18d4a157af162b375619a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c06c5e5ba900640d4af4339b4c0d6382a9b92eb33893ea9ee574fe98c77d15e +size 2845352958 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..eecf48236b3c9c0f22d0872bd429512ae532889a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dffa6b0d7841d05da93455ac335855679e1af47c238f8fe8032ceb383e2bebd0 +size 2845316241 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5dbefb696dd5e6aad3988aa6d2530647891c5750 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bef5bbf95a7c0ccdd1db3ca962b72387e9604c873a5d7ed3b4858e9ed2f5a63 +size 635546 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fe8c369f9d8481dfdad76fe11a40fd9b0fc5750e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4438dfc73d1126b56bfd66d4caef813b5b4c5d91e3544d3eef6bf3c7907c8a52 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0ff63552547fcf8601d5fbddf1273d094265c3d1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a8e47ade94ca4cc2df5bd9485c49af59cc6e8252c335d5a8ea349c77f8b26dc +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..297c4d64807f1a115a12df61f031b9383cb49d32 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2af4af59ef6c72259b38f73e01692ed4d123d4f763f1489429b5d802ad109b9c +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0b4c12fc3a04cd9f4e13e2b22963616f66012e43 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4fd540559f238077860a1405172a7f5fbd64febb531b4cd49242e5be2cbfa19 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..518b0d6ec912a93f9997d88040472f2d83fa3b78 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65b50486b96cfc682bc9f0410cffed9e269533ac874e8035f74c55ca2b2e9e5c +size 1722760 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6945a63407420756a455039eac95f8eaed3cd896 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fcbe15edb92932940ee5dc4331d5eb1efde40d4873e782dfce8bd657326245f +size 2847149355 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..25d670f84f541d16dfbe2525c006350eeb2a1fca --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc2b89a200976be51afd7a102a772a0bc0abe9ef8e9a4e40e91f3b814ffcdcf0 +size 2845267132 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..03de8f642ee1d8c23f72a95bf9c4c53060becc92 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ead037af6ebeef19a5d59e5bd4673048e19a252b91293cc355ac7e5a00e0041e +size 2845352958 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a2115860134a3057d1b614f14d35f3eca4e4b392 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea96d6b1cfcc0ce655c3a2aaf072d97cd8048133867db85e03f78b28c52adf75 +size 2845316241 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.pth new file mode 100644 index 0000000000000000000000000000000000000000..97cb41dfed21f440317a176863edebc119d1a324 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c1c25fd6c3ea2819196abcf7cba095dde0d86545b36426cd731e4feee3caa8 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/config.yaml b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18d4654e1b2290f00c24097a46cfe1c532c21149 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_2step_t833_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'False' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '0' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.833' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '600' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/debug-internal.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3663c8754fb65fe7b4f46a742480ac18d70d6db9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-03T20:16:15.915170849+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-03T20:16:16.376091891+09:00","level":"INFO","msg":"stream: created new stream","id":"pdowrk23"} +{"time":"2026-05-03T20:16:16.376151425+09:00","level":"INFO","msg":"handler: started","stream_id":"pdowrk23"} +{"time":"2026-05-03T20:16:16.377409672+09:00","level":"INFO","msg":"stream: started","id":"pdowrk23"} +{"time":"2026-05-03T20:16:16.377430695+09:00","level":"INFO","msg":"writer: started","stream_id":"pdowrk23"} +{"time":"2026-05-03T20:16:16.377434325+09:00","level":"INFO","msg":"sender: started","stream_id":"pdowrk23"} +{"time":"2026-05-04T08:05:40.278779245+09:00","level":"INFO","msg":"stream: closing","id":"pdowrk23"} +{"time":"2026-05-04T08:05:41.842704517+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-04T08:05:42.103414303+09:00","level":"INFO","msg":"handler: closed","stream_id":"pdowrk23"} +{"time":"2026-05-04T08:05:42.103940989+09:00","level":"INFO","msg":"sender: closed","stream_id":"pdowrk23"} +{"time":"2026-05-04T08:05:42.10395477+09:00","level":"INFO","msg":"stream: closed","id":"pdowrk23"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/debug.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..86e4b7084559e2468af7896381713036a114c767 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/debug.log @@ -0,0 +1,21 @@ +2026-05-03 20:16:15,613 INFO MainThread:2319385 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-03 20:16:15,613 INFO MainThread:2319385 [wandb_setup.py:_flush():81] Configure stats pid to 2319385 +2026-05-03 20:16:15,614 INFO MainThread:2319385 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-03 20:16:15,614 INFO MainThread:2319385 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug.log +2026-05-03 20:16:15,614 INFO MainThread:2319385 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug-internal.log +2026-05-03 20:16:15,615 INFO MainThread:2319385 [wandb_init.py:init():844] calling init triggers +2026-05-03 20:16:15,615 INFO MainThread:2319385 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_2step_t833_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '600', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.833', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-03 20:16:15,615 INFO MainThread:2319385 [wandb_init.py:init():892] starting backend +2026-05-03 20:16:15,899 INFO MainThread:2319385 [wandb_init.py:init():895] sending inform_init request +2026-05-03 20:16:15,912 INFO MainThread:2319385 [wandb_init.py:init():903] backend started and connected +2026-05-03 20:16:15,916 INFO MainThread:2319385 [wandb_init.py:init():973] updated telemetry +2026-05-03 20:16:15,935 INFO MainThread:2319385 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-03 20:16:17,104 INFO MainThread:2319385 [wandb_init.py:init():1042] starting run threads in backend +2026-05-03 20:16:17,351 INFO MainThread:2319385 [wandb_run.py:_console_start():2524] atexit reg +2026-05-03 20:16:17,351 INFO MainThread:2319385 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-03 20:16:17,351 INFO MainThread:2319385 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-03 20:16:17,351 INFO MainThread:2319385 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-03 20:16:17,357 INFO MainThread:2319385 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-04 08:05:40,278 INFO wandb-AsyncioManager-main:2319385 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-04 08:05:40,279 INFO wandb-AsyncioManager-main:2319385 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/config.yaml b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d002c608a6db1de75eb62d8f140fe0cb7a3d3d4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/config.yaml @@ -0,0 +1,388 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + x345s6oebbth4ne5yw7ylqsrwuq0vonf: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_2step_t833_no_reward.py + - '-' + - trainer.resume=False + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_2step_t833_no_reward + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1175529656320" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: b73d18c2e78b2465b81e9263c2223e336a1ea136 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward + startedAt: "2026-05-03T11:16:15.596336Z" + writerId: x345s6oebbth4ne5yw7ylqsrwuq0vonf + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_2step_t833_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "False" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "0" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.833" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "600" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9684c7bde9ed18086d39e7a9880b642c505b3947 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112c0905448a821974f0bf0a94d9107473bb02b60e0ebe1e75a57d309670facb +size 219311 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e38036c10e966c2c82fa717b29431a1ff88e1ae --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae +size 190461 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_100_02a9469e6a8f62c40ad7.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_100_02a9469e6a8f62c40ad7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb07f96488299ea719e7a7ed02f378f197ba3a60 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_100_02a9469e6a8f62c40ad7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02a9469e6a8f62c40ad773112921999cd21f82dcb70c74c5e19c79231bad6d0f +size 187234 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_200_2af0f38fdf389dae8d28.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_200_2af0f38fdf389dae8d28.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..454bfe56541a9c7101f4882b72ed49fecdaafba5 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_200_2af0f38fdf389dae8d28.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2af0f38fdf389dae8d288de9f5b381c7389b74f4006a7494fa17c0edc51fc8a5 +size 185108 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_300_38c363a3e652a07dc1bb.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_300_38c363a3e652a07dc1bb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3540944a715be42d3d84741b9017ba7ff863e3e8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_300_38c363a3e652a07dc1bb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38c363a3e652a07dc1bbd11efd86e21a02938249047fda441293b3501c920d21 +size 208799 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_400_16d8fbd2f6c73ac8872f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_400_16d8fbd2f6c73ac8872f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6886321df5b3a9c23cccb421621a24d6a67a7052 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_400_16d8fbd2f6c73ac8872f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d8fbd2f6c73ac8872fa6686e738d46377a15c7523a36ed01819528f0f2fab9 +size 229937 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_500_7abed1a68789457ad3a0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_500_7abed1a68789457ad3a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5b0a0b7ef900f6bcbc80937cb53678a7c9aa79e2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/train_media/student/generation_500_7abed1a68789457ad3a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7abed1a68789457ad3a077eca4385658ff06eb942ae3ed10f63dbfe4da5df3bb +size 207530 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_1fc3b5e201154e6e34af.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_1fc3b5e201154e6e34af.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5e622ddfec939fefcdb4e7524cc54ed8f07df05 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_1fc3b5e201154e6e34af.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fc3b5e201154e6e34af2ea13adecd85d7d10b7481bd3a37a22e32ab2a832748 +size 215095 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_24922a8666c073275c9a.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_24922a8666c073275c9a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9017cad729f098eedeb68cc71f1d8ea74758454b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_24922a8666c073275c9a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24922a8666c073275c9af64c43195f7253e96200b964427e478956360773cc36 +size 209877 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_4fcbdc3dc61e38daa972.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_4fcbdc3dc61e38daa972.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6dc7d19848efc1dfae267a1226a41160e851f197 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_4fcbdc3dc61e38daa972.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fcbdc3dc61e38daa9722be774f8e2f411ad01b297aade099c0a19e363c24f5f +size 244371 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_62bed84a4aa382a8a841.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_62bed84a4aa382a8a841.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e38d97b594cd45c580cc85c07fd9ff14935ae6f0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_62bed84a4aa382a8a841.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62bed84a4aa382a8a841e56715d46e054159e3361efc2c34271fcafc1bbe6d35 +size 212739 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_6986716832066cc0b4b3.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_6986716832066cc0b4b3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62ff89d8cad5e113ccaf26cf52f53b5c2cf905a4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_6986716832066cc0b4b3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6986716832066cc0b4b3c4b9fc0ca80fd546e3bc39f2c80fb02d959ae10a8c96 +size 176196 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_989042d0d296c8757b9c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_989042d0d296c8757b9c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e6eb52cb9883ed19592dc202f7d00b357f4eb95 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_989042d0d296c8757b9c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:989042d0d296c8757b9c5fe9f1f88dd4dec178a261ec5c06b7a40f685e460c10 +size 181107 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_a311ef087e615abc4b88.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_a311ef087e615abc4b88.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8a6a6dd3395e82181a320c8a7caf24791121b8d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_a311ef087e615abc4b88.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a311ef087e615abc4b88701428b8da49c1fc7258d5df3a083d2be6fbe8e0e151 +size 160707 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_a9ee1f7ea80e20a77ba8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_a9ee1f7ea80e20a77ba8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..865448723986b6a666886e9522afa6e6d8dd9bb4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_a9ee1f7ea80e20a77ba8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9ee1f7ea80e20a77ba8edcff1f2aabf64fef090815706e054247a8cc82095ef +size 188470 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_cd952d64b92f2ae55e8d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_cd952d64b92f2ae55e8d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0949822395b15fce484ec2719d8354f30faa4e6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_cd952d64b92f2ae55e8d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd952d64b92f2ae55e8d3021ce604bbcfe436560a9e55d964ba1ff62dda17576 +size 169856 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_e7625fd6dbecfcd47200.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_e7625fd6dbecfcd47200.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49ccc4d1f6b7ee75a85eed1c744578fd85a19c44 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_100_e7625fd6dbecfcd47200.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7625fd6dbecfcd4720094b2921bbda2ec77fa670214660020f6c38f69ed5fba +size 260787 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_13b0ba1edef33da14281.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_13b0ba1edef33da14281.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..04c0b6918f8b3e34bcd10fa63c5499176d2bc3eb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_13b0ba1edef33da14281.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b0ba1edef33da14281e0d3dbef94674afdfc2a86f2f854855a3dfc25f84a7c +size 190437 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_23962546ea0008325d27.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_23962546ea0008325d27.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6cc6335e76adc601fc4cf66e583be60b14d272ab --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_23962546ea0008325d27.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23962546ea0008325d277da5db7d9e4a045750a8a3b063a0680bd6b4fb35bf07 +size 217640 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_44858b0cbf80e0776333.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_44858b0cbf80e0776333.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cfa817d220cd08cb7e1266ac7c812b9e0ff53619 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_44858b0cbf80e0776333.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44858b0cbf80e0776333408c4a0cb174cf5b00a5fa427f55ba17443e09344f22 +size 248863 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_514ead401e4d4d48563b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_514ead401e4d4d48563b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4495a19f5c68ddc164bd201d2643715999f46e77 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_514ead401e4d4d48563b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:514ead401e4d4d48563bd10b9466ce37f2ba236f46afd83abdfb08e47072c670 +size 178391 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_76d85fc2939323206846.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_76d85fc2939323206846.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..21d822c84eb941924d66c70d455df970588a4727 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_76d85fc2939323206846.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76d85fc2939323206846965892f8a51fcdc0cb39eaf3dc97e3897b9955996987 +size 219006 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_7d946e28fdb7615e3a75.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_7d946e28fdb7615e3a75.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05ff1a7e6ae55b5a7d2441ae9d8d1530c0fb1c10 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_7d946e28fdb7615e3a75.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d946e28fdb7615e3a7529db7c98f78e36c2791aece98e3d4a668eae05ef0b82 +size 174698 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_94dc14699d4b02dcf3e4.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_94dc14699d4b02dcf3e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8c7fa3512cfea78419d5734e371915ff4228dc3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_94dc14699d4b02dcf3e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94dc14699d4b02dcf3e4e042040023a4d1bc8b8ab5a530857efe78b29b6f0146 +size 185737 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_9527a798369191a957ee.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_9527a798369191a957ee.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c35879d2bb486206f4286f5ed224bd4acb29eaea --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_9527a798369191a957ee.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9527a798369191a957ee072eb41d247013478f6ea542ef65535c706e077daf3a +size 262594 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_a016c316831a3184c893.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_a016c316831a3184c893.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..12aca05a29aefe8da8789e7633b6eafd59ed92d0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_a016c316831a3184c893.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a016c316831a3184c893990d6fa09d2b7c4958bbc496ea971b22bedb888e535f +size 213698 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_d398a1b31d3dd0daddaf.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_d398a1b31d3dd0daddaf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d19a1f8aab56d2ff1545a8ce46452c5400e646d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_200_d398a1b31d3dd0daddaf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d398a1b31d3dd0daddafba86fc1e7ad82eb6c90eb8e1ac25f89760b041134b35 +size 163406 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_104a15bb2c320b7cd460.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_104a15bb2c320b7cd460.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..afca6890c9dcae2c77e93bc40c04e67e277803c7 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_104a15bb2c320b7cd460.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:104a15bb2c320b7cd46002abb2cff016bcb74b474da0c964ac69f5cea00daf10 +size 187042 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_2d21142e2c65dd101874.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_2d21142e2c65dd101874.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..43294b3fd79daeb88f2f7163b06e9873882e9a3d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_2d21142e2c65dd101874.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d21142e2c65dd1018747112004c4b5bee0edac6cbb2cd0e9eeb8537b5e9ed8c +size 192942 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_89aa3c5736b12b29c461.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_89aa3c5736b12b29c461.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05f980b7c5edf9f3bf0074c9496adcb6db7e014f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_89aa3c5736b12b29c461.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89aa3c5736b12b29c46199432e6122ee4600a5025bdfe7911cbf5cfee3f756cc +size 284307 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_8ca35e1e0f4c87746fcb.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_8ca35e1e0f4c87746fcb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f13cc691494cbb561c06bf7a4dc28a7af7950d78 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_8ca35e1e0f4c87746fcb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ca35e1e0f4c87746fcbd4ab9e3a15ba3b86bff52cf99e43ace5839a1bab93ba +size 174046 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_989687e266022d183500.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_989687e266022d183500.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e96f17540aaff8fc5e83bb35554acab45d5ff87d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_989687e266022d183500.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:989687e266022d1835008b54d0f3b85063e1ff30c705fd179f8fb0d5b41efcbd +size 230994 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_9f19fcc77f53bceef388.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_9f19fcc77f53bceef388.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b1e35ea436b0ec2b08b92833ef9d30d9fe9ba9b8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_9f19fcc77f53bceef388.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f19fcc77f53bceef388a505e474fc246b4c57a7645b25f5ba68a448241bbf8b +size 202388 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_ca18021f1758bd6e5e2f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_ca18021f1758bd6e5e2f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..60abf90f583d7bd68dc6bc02e162e585ffdbc4cf --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_ca18021f1758bd6e5e2f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca18021f1758bd6e5e2fd6bf78c4eb8a3de0677cd4b06d2b96d2e5a6e19f7fc6 +size 201781 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_d348cf69473e92cd8bb9.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_d348cf69473e92cd8bb9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49eb46a87cf4e840d269b84a645d0f05a23130d6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_d348cf69473e92cd8bb9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d348cf69473e92cd8bb9ec737a0b251722566f715c8861f681b9f2810fe8b4da +size 225383 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_e61d818001c341ef6fcd.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_e61d818001c341ef6fcd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2793a9d321dfaee96c8463ad0c3188cc80e4b5bf --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_e61d818001c341ef6fcd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e61d818001c341ef6fcd56aad27b294eb1630358d4844bc586a0a9388e8efbd2 +size 234407 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_f77db77f9edd6deeea55.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_f77db77f9edd6deeea55.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..151cc583f4f6c2dc29c521e37de2df4aad2c271a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_300_f77db77f9edd6deeea55.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f77db77f9edd6deeea5507ad059f30eb243aa5d85bc53ee1a5bf14ec3f213511 +size 267845 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_02fc4d5308d7410b9769.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_02fc4d5308d7410b9769.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f060643e263c67fa02e49d664fec1cf0d5d123b6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_02fc4d5308d7410b9769.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02fc4d5308d7410b9769fa5b670ffb60c377e0675f5f4931ec8341bf3a3c02d4 +size 183798 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_0af5a053a398519cd953.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_0af5a053a398519cd953.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..511289ae590b6e28a807cb80ffde14ca6f57a3ef --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_0af5a053a398519cd953.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0af5a053a398519cd9531d3dd7667544f31a566ec7a948744ccb5d0e4112813f +size 276335 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_12c1b7ac5a908057071a.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_12c1b7ac5a908057071a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ed54188e4ccaa89070195eb778d99eba460a48cd --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_12c1b7ac5a908057071a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12c1b7ac5a908057071a37368ded349d91c7e09c562c200e80aa43ae921b268d +size 227085 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_15143e2dc19672250f8c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_15143e2dc19672250f8c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9102731055fcb15698d48368b677ad68ec7b93b3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_15143e2dc19672250f8c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15143e2dc19672250f8c333617faea8021e2ba34019fd27c4af1cb737011595c +size 198622 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_1f229e284511cd47616d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_1f229e284511cd47616d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..71b0514c80f22931116e471638fb6557acaa0926 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_1f229e284511cd47616d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f229e284511cd47616d38f40d7409ec0e6becf69552a80992dddc259082c7d2 +size 232094 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_8fd7e644dfc8020f3337.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_8fd7e644dfc8020f3337.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bbd5077a8c89dfd93a6c8030a721625beefa0d6e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_8fd7e644dfc8020f3337.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fd7e644dfc8020f3337c05dcaf9df6cc1b5e8bca68bce44e6486bdd18e03c9b +size 200411 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_95156dbf3f0b2df3db0c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_95156dbf3f0b2df3db0c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b481cb5523a1f29c4d2312f8640501c593619576 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_95156dbf3f0b2df3db0c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95156dbf3f0b2df3db0c9c3c160966a041c2d2989967120c388c785ed7c906d4 +size 173938 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_a8cd1b085302aefed74e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_a8cd1b085302aefed74e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a084e3ec9473e646eb52bbb7de86fa3f53cdb5b5 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_a8cd1b085302aefed74e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8cd1b085302aefed74eddd401aa1ea79000a4afd678555df0d678d193ef5b53 +size 191864 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_b993b55b7abeeda934f0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_b993b55b7abeeda934f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1460a04f746cf160fd67353df6c5679826b1dacd --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_b993b55b7abeeda934f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b993b55b7abeeda934f05d1c1e6d3e2adee3cf71b16f250671a89f13fb2e669f +size 267164 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_d7dbecce91d9e83ae5e1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_d7dbecce91d9e83ae5e1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b7cde9b77362c12c55850fd56b1ea37e8f390e36 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_400_d7dbecce91d9e83ae5e1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7dbecce91d9e83ae5e1760b431190365f918f561b964aad5103eb24d3356027 +size 222653 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_00ffc87bb9b6e918112e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_00ffc87bb9b6e918112e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67578e787484eec00f00b69db0ca8563b0393741 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_00ffc87bb9b6e918112e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00ffc87bb9b6e918112e64bd1a631be8a6b26c4414d86ab79fcf5449bf766bf5 +size 284059 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_3d30c382480ad9b7821c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_3d30c382480ad9b7821c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2773daaa5a09b6c0dfb490a6916d10e164461622 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_3d30c382480ad9b7821c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d30c382480ad9b7821c2c70f34d27d29565b53466ecf04dc4e3d932294dd0a3 +size 194779 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_3d83faf19efb58bd1ca3.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_3d83faf19efb58bd1ca3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61aa5c15f3c01c162f39211ce1aff548c59d39d3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_3d83faf19efb58bd1ca3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d83faf19efb58bd1ca3c34ca04128b7e31933fa9944a572bd86e677a821365f +size 280673 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_77b86d717bc4e6ae54de.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_77b86d717bc4e6ae54de.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b9d01fbea8c3cf262d25f1f225f2741020e2401 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_77b86d717bc4e6ae54de.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77b86d717bc4e6ae54deb9816bb6a0fc2bc2bba0ff5a2c4001e52becb9d02781 +size 179388 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_921361a9f8134e388fa7.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_921361a9f8134e388fa7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5db2e902290505d68202c2c2d8cfb6d51e4eb936 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_921361a9f8134e388fa7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:921361a9f8134e388fa7e4229eb47c82674743b27904e9f3dea3184d18daa2cb +size 232797 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_92cff2dca642f0ba9d81.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_92cff2dca642f0ba9d81.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..803dc0bda595469c53b15a7b050b58a1d4fed45c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_92cff2dca642f0ba9d81.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92cff2dca642f0ba9d81a2b32529c6aca65f3a9758e62ae6e8d695847635a39a +size 206130 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_9e98e7916427159cdbe5.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_9e98e7916427159cdbe5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9c3e85474ff46a7b1dfc4b7d1d8b2bd9bd22142 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_9e98e7916427159cdbe5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e98e7916427159cdbe5894b9a1be9f7dc9cdaa6b9f29b218b3f0de2acbf0a4c +size 227711 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_cb02a559e7336c93031b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_cb02a559e7336c93031b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23efff733f19bf16320fffce8ed194a3b8677609 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_cb02a559e7336c93031b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb02a559e7336c93031bd302a8a91a0c89f3c74b3471ad73af5fca66a4338792 +size 240173 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_db8c7492f671ca319ef5.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_db8c7492f671ca319ef5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..19f1e81250a8928d74339b6643ebf550b2ebdb20 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_db8c7492f671ca319ef5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db8c7492f671ca319ef501912c5585a82031d14c146dc2dc5d021417b7c2607a +size 187865 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_f88305a649c54fbcf67c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_f88305a649c54fbcf67c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..213444d27160ca4eb9bfd852de5a71537582cba1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_500_f88305a649c54fbcf67c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f88305a649c54fbcf67c194b9c5b561cb5100db9ec9144552e6e0e69a18f45cf +size 204436 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_16b78933092d6668cdb7.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_16b78933092d6668cdb7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad259703955ab33adafb6a11d84257428de435fa --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_16b78933092d6668cdb7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16b78933092d6668cdb7741ac37d4e4db82dcb646d991bbbe0f99057c759f21b +size 181424 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_46424061a7c159bc5d15.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_46424061a7c159bc5d15.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08a30443c8145f60821b8ded7fdd0857e2304fe7 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_46424061a7c159bc5d15.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46424061a7c159bc5d154dbe9d6cab657dc963d2c99a122663b316684dcd0218 +size 200472 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_56348bb82660a9b2298b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_56348bb82660a9b2298b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca4aa3e525e890f821a7c64a5d3336717d18eae6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_56348bb82660a9b2298b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56348bb82660a9b2298b618f3b887d1cdeae6b1e873f98daef5a062ae053d0d5 +size 287661 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_57748de0546e2eca3715.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_57748de0546e2eca3715.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4e8393eea6ec57e81e9c7fd14eb3e7b21a8c218 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_57748de0546e2eca3715.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57748de0546e2eca3715b13813ece5ec42308930a22ef2e1e49846295311713f +size 246160 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_5fad37a16041bbf6fa85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_5fad37a16041bbf6fa85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9b9b059a11453c0a96c63cb205153c49de78c48 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_5fad37a16041bbf6fa85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fad37a16041bbf6fa858dd43a81f2558f6e96f5e742fae3cfe6a11b204e182b +size 286681 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_8cadb1fdd841ccef6984.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_8cadb1fdd841ccef6984.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7092ea8be20dfcb812ad5bb78183d29a421297e1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_8cadb1fdd841ccef6984.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cadb1fdd841ccef69841f1f7c5e212b76bd51fd9e3407aee2f3de7620a95f07 +size 238718 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_9dd369241b0d624306b6.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_9dd369241b0d624306b6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c918e5e3d98729b1efbb89f858587226175a9405 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_9dd369241b0d624306b6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd369241b0d624306b6dd1978e702fd50e2b33c4919cb74959048c61b5b6863 +size 205906 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_d4cae118f95943b9957a.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_d4cae118f95943b9957a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..938317cafb3d170f213215eb5d325158b75558b0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_d4cae118f95943b9957a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4cae118f95943b9957ac5437416f2256fdaab52e79240c7284b4a8ac09c34ee +size 207927 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_dfdfdd14726cd8f9b0e4.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_dfdfdd14726cd8f9b0e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4a2ab0a16d8465a4a058f693e32a5004cc069e87 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_dfdfdd14726cd8f9b0e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfdfdd14726cd8f9b0e4c003a910610f887000e6b592766c7e40d94a3d1b99e5 +size 190962 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_e3853e4d45d36ffc859e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_e3853e4d45d36ffc859e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..baea20fa69fd10770faf569a33d1fcf97f6b6e58 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/generated_600_e3853e4d45d36ffc859e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3853e4d45d36ffc859e3792969cfe5fb123d1fa52d8c8f88d243e216ed62dbb +size 233977 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/output.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3327396e7b176b5fd732e359de93e278fe38dc31 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/output.log @@ -0,0 +1,10736 @@ +[May 3, 2026 - 20:16:17 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 3, 2026 - 20:16:17 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 3, 2026 - 20:16:17 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 3, 2026 - 20:16:17 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 3, 2026 - 20:16:17 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 3, 2026 - 20:16:17 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 3, 2026 - 20:16:17 | INFO | fastgen.trainer:run:77 ] Starting training +[May 3, 2026 - 20:16:17 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 3, 2026 - 20:16:17 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 3, 2026 - 20:16:37 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 3, 2026 - 20:16:57 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 3, 2026 - 20:16:57 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 3, 2026 - 20:16:57 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 3, 2026 - 20:16:57 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 3, 2026 - 20:18:07 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 3, 2026 - 20:18:07 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 3, 2026 - 20:18:07 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 3, 2026 - 20:18:07 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 3, 2026 - 20:18:07 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 3, 2026 - 20:18:07 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 3, 2026 - 20:18:10 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 3, 2026 - 20:18:10 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.6s +[May 3, 2026 - 20:18:10 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 3, 2026 - 20:18:10 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 3, 2026 - 20:18:12 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 3, 2026 - 20:18:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 3.1s +[May 3, 2026 - 20:18:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 3, 2026 - 20:18:13 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 3, 2026 - 20:18:28 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 3, 2026 - 20:18:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 16.2s +[May 3, 2026 - 20:18:29 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 3, 2026 - 20:18:29 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 3, 2026 - 20:18:29 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 3, 2026 - 20:18:29 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[May 3, 2026 - 20:18:29 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 3, 2026 - 20:18:36 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 3, 2026 - 20:18:36 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 3, 2026 - 20:18:36 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 3, 2026 - 20:18:36 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17136.85 M total params (logical). +[May 3, 2026 - 20:18:36 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4285.56 M total params LOCAL on rank 0. +[May 3, 2026 - 20:18:36 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 1. +[May 3, 2026 - 20:18:36 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 2. +[May 3, 2026 - 20:18:36 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 3. +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 3, 2026 - 20:18:37 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 3, 2026 - 20:18:37 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.393344 37.445824 37.347359 +peak_gpu_mem_gb 57.011160 57.011160 57.011160 +peak_gpu_mem_reserved_gb 59.156250 59.156250 59.156250 +util 79.500000 86.000000 75.000000 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.21 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 10.75 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 11.21 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.81 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:19:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.06 seconds +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.569074 37.621662 37.523067 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 92.750000 99.000000 87.000000 +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 52.04 seconds +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 13.42 +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 11.79 +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 1.74 +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:20:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.569304 37.621880 37.523457 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 99.500000 100.000000 98.000000 +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 55.17 seconds +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 15.21 +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 11.16 +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 1.79 +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:21:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.569718 37.621948 37.523808 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.211426 61.212891 61.210938 +util 97.500000 100.000000 95.000000 +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 48.95 seconds +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 11.79 +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 11.78 +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 1.78 +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:22:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.591272 37.642132 37.545540 +peak_gpu_mem_gb 96.121632 96.123194 96.121111 +peak_gpu_mem_reserved_gb 102.568848 102.658203 102.539062 +util 67.500000 90.000000 53.000000 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 178.55 seconds +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 79.25 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 10.20 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.45 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8555 iter count: 1.0 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8555 iter count: 1.0 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8555 iter count: 1.0 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8555iter count: 1.0 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:25:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.591502 37.642334 37.545891 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.758789 64.820312 64.738281 +util 95.750000 100.000000 89.000000 +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 55.35 seconds +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 15.31 +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 11.27 +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 1.64 +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.591997 37.642830 37.546318 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656250 64.703125 64.640625 +util 94.000000 100.000000 90.000000 +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 55.24 seconds +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 14.96 +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 11.73 +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 1.78 +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:26:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.592154 37.643227 37.546513 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.631836 64.664062 64.621094 +util 96.750000 100.000000 89.000000 +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 54.14 seconds +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 14.71 +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 11.22 +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 1.69 +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 8-------------------- +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:27:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.592316 37.643360 37.546524 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.714844 64.761719 64.699219 +util 96.000000 99.000000 90.000000 +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 46.15 seconds +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 11.33 +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 10.13 +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 1.57 +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 9-------------------- +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:28:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.592922 37.643585 37.547321 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.728027 105.882812 105.263672 +util 67.750000 72.000000 62.000000 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 173.82 seconds +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 77.12 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 9.43 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.40 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 10-------------------- +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8311 iter count: 1.0 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8311 iter count: 1.0 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8311 iter count: 1.0 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8311iter count: 1.0 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:31:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.592999 37.643673 37.547321 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.680664 64.859375 64.621094 +util 97.250000 100.000000 94.000000 +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 53.59 seconds +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 14.17 +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 11.84 +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 1.80 +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 11-------------------- +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:32:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.593824 37.644772 37.547523 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.651367 64.742188 64.621094 +util 96.500000 99.000000 94.000000 +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 54.96 seconds +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 15.08 +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 11.15 +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 1.79 +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 12-------------------- +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0062 iter count: 1.0 +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:33:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.594080 37.644772 37.547523 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.661133 64.781250 64.621094 +util 94.500000 100.000000 89.000000 +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 55.30 seconds +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 14.95 +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 11.81 +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 1.78 +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 13-------------------- +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:34:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.594283 37.645157 37.547535 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.622070 64.742188 64.582031 +util 94.000000 100.000000 87.000000 +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 50.00 seconds +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 12.86 +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 11.33 +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 1.80 +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 14-------------------- +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:34:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.594853 37.644760 37.549564 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.186035 105.263672 105.160156 +util 73.500000 95.000000 65.000000 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 177.30 seconds +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 78.66 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 10.23 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.39 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 15-------------------- +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7617 iter count: 1.0 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7617 iter count: 1.0 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7617 iter count: 1.0 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7617iter count: 1.0 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:37:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.595358 37.646675 37.549606 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 64.976562 64.875000 +util 97.250000 99.000000 95.000000 +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 49.81 seconds +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 12.48 +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 11.80 +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 1.79 +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 16-------------------- +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:38:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.595387 37.646687 37.549625 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871094 64.917969 64.855469 +util 99.000000 100.000000 98.000000 +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 51.24 seconds +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 13.28 +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 11.79 +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 1.79 +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 17-------------------- +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:39:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.595677 37.646973 37.550480 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749023 64.898438 64.699219 +util 95.000000 100.000000 88.000000 +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 51.30 seconds +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 13.27 +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 11.80 +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 1.80 +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 18-------------------- +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:40:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.595984 37.648216 37.550484 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.851562 64.898438 64.835938 +util 94.000000 99.000000 88.000000 +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 46.26 seconds +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 12.18 +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 10.15 +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 1.58 +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 19-------------------- +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0062 iter count: 1.0 +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:41:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596101 37.647919 37.550667 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.195801 105.302734 105.160156 +util 73.000000 88.000000 66.000000 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 174.43 seconds +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 77.27 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 9.43 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.40 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 20-------------------- +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.9092 iter count: 1.0 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.9092 iter count: 1.0 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.9092 iter count: 1.0 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.9092iter count: 1.0 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:44:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596252 37.648327 37.550709 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.807617 64.898438 64.777344 +util 95.750000 100.000000 89.000000 +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 49.86 seconds +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 12.55 +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 11.72 +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 1.56 +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 21-------------------- +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0058 iter count: 1.0 +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:44:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596283 37.648327 37.550770 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.675781 64.757812 64.429688 +util 94.750000 100.000000 88.000000 +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 52.23 seconds +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 13.74 +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 11.80 +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 1.79 +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 22-------------------- +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:45:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596293 37.648327 37.550751 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.617188 64.679688 64.429688 +util 92.750000 99.000000 86.000000 +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 51.02 seconds +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 13.28 +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 11.05 +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 1.59 +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 23-------------------- +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0060 iter count: 1.0 +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:46:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596389 37.648373 37.550812 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.602539 64.660156 64.429688 +util 95.500000 100.000000 92.000000 +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 51.24 seconds +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 13.26 +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 11.82 +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 1.78 +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 24-------------------- +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:47:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596616 37.648964 37.550983 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.093262 105.126953 105.082031 +util 65.250000 70.000000 61.000000 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 175.83 seconds +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 78.24 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 9.38 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.39 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 25-------------------- +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8057 iter count: 1.0 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8057 iter count: 1.0 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8057 iter count: 1.0 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8057iter count: 1.0 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:50:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596641 37.648968 37.551006 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.792969 64.839844 64.777344 +util 94.250000 100.000000 87.000000 +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 50.09 seconds +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 12.73 +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 11.04 +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 1.79 +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 26-------------------- +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:51:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596674 37.648945 37.551041 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.675781 64.722656 64.660156 +util 97.750000 100.000000 96.000000 +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 54.32 seconds +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 14.81 +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 11.16 +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 1.74 +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 27-------------------- +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:52:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596749 37.648968 37.551235 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 65.833008 66.222656 64.664062 +util 96.000000 99.000000 90.000000 +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 51.60 seconds +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 14.20 +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 10.13 +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 1.58 +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 28-------------------- +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:53:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596819 37.649258 37.551228 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.641602 64.644531 64.640625 +util 93.500000 100.000000 87.000000 +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 47.70 seconds +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 12.27 +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 11.27 +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 1.73 +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 29-------------------- +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:53:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596594 37.648079 37.551258 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.181152 105.244141 105.160156 +util 73.000000 80.000000 69.000000 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 179.55 seconds +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 80.56 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 9.15 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.39 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 30-------------------- +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8682 iter count: 1.0 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8682 iter count: 1.0 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8682 iter count: 1.0 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8682iter count: 1.0 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596669 37.648212 37.551388 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.835938 64.761719 +util 92.500000 99.000000 89.000000 +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 49.85 seconds +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 12.54 +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 11.39 +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 1.57 +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 31-------------------- +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:57:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596935 37.649220 37.551388 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.666016 64.683594 64.660156 +util 93.000000 100.000000 89.000000 +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 52.01 seconds +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 13.31 +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 11.80 +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 1.79 +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 32-------------------- +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:58:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596923 37.649178 37.551388 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.773438 64.796875 64.703125 +util 93.750000 99.000000 87.000000 +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 47.29 seconds +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 11.89 +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 10.90 +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 1.58 +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 33-------------------- +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 20:59:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596993 37.649460 37.551388 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.758789 64.777344 64.703125 +util 93.250000 99.000000 89.000000 +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 54.92 seconds +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 14.75 +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 11.81 +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 1.78 +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 34-------------------- +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0062 iter count: 1.0 +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:00:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596691 37.648224 37.551395 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.156738 105.205078 105.140625 +util 64.500000 68.000000 59.000000 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 179.49 seconds +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 79.77 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 9.76 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.40 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 35-------------------- +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7686 iter count: 1.0 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7686 iter count: 1.0 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7686 iter count: 1.0 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7686iter count: 1.0 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596690 37.648220 37.551395 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.793457 64.796875 64.783203 +util 99.000000 100.000000 98.000000 +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 53.41 seconds +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 13.98 +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 11.83 +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 1.79 +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 36-------------------- +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:04:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596901 37.648952 37.551395 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.739746 64.757812 64.685547 +util 96.750000 99.000000 95.000000 +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 48.56 seconds +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 11.67 +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 11.62 +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 1.79 +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 37-------------------- +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:04:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597031 37.649452 37.551395 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.759277 64.777344 64.705078 +util 94.750000 99.000000 87.000000 +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 48.87 seconds +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 13.45 +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 10.22 +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 1.57 +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 38-------------------- +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:05:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597048 37.649452 37.551399 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.798340 64.816406 64.744141 +util 89.250000 100.000000 72.000000 +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 49.64 seconds +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 13.30 +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 10.14 +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 1.70 +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 39-------------------- +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:06:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.02 seconds +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596844 37.648643 37.551407 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.215332 105.263672 105.199219 +util 64.000000 73.000000 56.000000 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 165.81 seconds +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 74.26 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 8.48 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.39 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 40-------------------- +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8027 iter count: 1.0 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8027 iter count: 1.0 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8027 iter count: 1.0 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8027iter count: 1.0 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:09:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596874 37.648670 37.551411 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.905762 64.939453 64.894531 +util 94.250000 99.000000 88.000000 +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 49.79 seconds +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 13.98 +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 10.13 +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 1.57 +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 41-------------------- +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:10:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597055 37.649418 37.551411 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.773926 64.875000 64.470703 +util 96.000000 99.000000 90.000000 +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 46.11 seconds +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 12.12 +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 10.14 +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 1.58 +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 42-------------------- +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:10:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597079 37.649418 37.551411 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.910645 64.958984 64.894531 +util 94.000000 100.000000 87.000000 +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 43.49 seconds +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 10.81 +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 10.14 +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 1.58 +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 43-------------------- +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:11:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597102 37.649441 37.551456 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.769043 64.875000 64.451172 +util 93.750000 100.000000 87.000000 +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 46.00 seconds +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 12.04 +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 10.15 +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 1.58 +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 44-------------------- +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:12:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596745 37.648010 37.551437 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.146973 105.160156 105.107422 +util 66.500000 75.000000 59.000000 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 161.03 seconds +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 71.91 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 8.41 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.39 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 45-------------------- +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7852 iter count: 1.0 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7852 iter count: 1.0 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7852 iter count: 1.0 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7852iter count: 1.0 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:15:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596820 37.648098 37.551617 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749512 64.835938 64.490234 +util 93.750000 98.000000 88.000000 +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 49.86 seconds +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 13.98 +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 10.15 +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 1.59 +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 46-------------------- +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:15:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596925 37.648548 37.551586 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.710449 64.796875 64.451172 +util 97.750000 100.000000 93.000000 +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 46.07 seconds +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 12.11 +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 10.13 +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 1.57 +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 47-------------------- +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:16:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597016 37.648819 37.551632 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.637207 64.699219 64.451172 +util 96.500000 99.000000 91.000000 +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 48.78 seconds +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 13.46 +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 10.14 +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 1.58 +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 48-------------------- +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:17:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597156 37.649288 37.551620 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.710449 64.796875 64.451172 +util 96.000000 100.000000 89.000000 +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 48.64 seconds +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 13.40 +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 10.13 +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 1.56 +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 49-------------------- +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:18:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596881 37.648064 37.551739 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.371582 105.771484 105.238281 +util 66.250000 71.000000 61.000000 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 158.00 seconds +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 70.37 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 8.41 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.43 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 50-------------------- +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7900 iter count: 1.0 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7900 iter count: 1.0 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7900 iter count: 1.0 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7900iter count: 1.0 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:21:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596895 37.648087 37.551750 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.836914 64.839844 64.835938 +util 97.500000 99.000000 93.000000 +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 47.18 seconds +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 12.68 +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 10.11 +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 1.57 +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 51-------------------- +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:21:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597010 37.648560 37.551758 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.705078 64.718750 64.664062 +util 97.250000 99.000000 92.000000 +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 45.99 seconds +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 12.06 +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 10.15 +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 1.58 +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 52-------------------- +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597001 37.648563 37.551716 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.827148 64.835938 64.800781 +util 96.250000 100.000000 93.000000 +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 46.14 seconds +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 12.15 +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 10.12 +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 1.56 +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 53-------------------- +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:23:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597004 37.648563 37.551716 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.773438 64.796875 64.703125 +util 92.500000 99.000000 89.000000 +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 46.15 seconds +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 12.14 +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 10.16 +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 1.58 +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 54-------------------- +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:24:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596820 37.647747 37.551716 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.146973 105.166016 105.140625 +util 64.500000 69.000000 59.000000 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 161.09 seconds +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 71.93 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 8.41 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.40 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 55-------------------- +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7588 iter count: 1.0 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7588 iter count: 1.0 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7588 iter count: 1.0 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7588iter count: 1.0 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:26:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596836 37.647739 37.551830 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.939453 65.015625 64.914062 +util 95.250000 100.000000 88.000000 +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 44.65 seconds +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 11.40 +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 10.14 +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 1.57 +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 56-------------------- +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:27:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596960 37.648235 37.551830 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.797852 64.894531 64.507812 +util 96.250000 99.000000 90.000000 +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 48.63 seconds +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 13.39 +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 10.14 +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 1.57 +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 57-------------------- +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:28:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597301 37.649574 37.551830 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.470703 64.488281 64.464844 +util 93.500000 98.000000 88.000000 +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 46.10 seconds +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 12.12 +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 10.15 +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 1.56 +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 58-------------------- +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:29:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597397 37.649841 37.551929 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.949219 64.953125 64.937500 +util 97.750000 100.000000 93.000000 +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 46.06 seconds +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 12.10 +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 10.17 +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 1.56 +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 59-------------------- +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:29:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596987 37.648094 37.551945 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.190918 105.283203 105.160156 +util 66.750000 72.000000 59.000000 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 161.03 seconds +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 71.90 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 8.41 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.40 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 60-------------------- +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7549 iter count: 1.0 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7549 iter count: 1.0 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7549 iter count: 1.0 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7549iter count: 1.0 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:32:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597020 37.648094 37.551945 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.881348 64.914062 64.783203 +util 95.750000 100.000000 93.000000 +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 49.89 seconds +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 14.02 +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 10.15 +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 1.56 +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 61-------------------- +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:33:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597155 37.648586 37.551945 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.891113 64.933594 64.763672 +util 97.500000 99.000000 93.000000 +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 46.08 seconds +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 12.10 +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 10.16 +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 1.58 +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 62-------------------- +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:34:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597466 37.649837 37.551956 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.866699 64.933594 64.666016 +util 93.500000 100.000000 90.000000 +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 43.49 seconds +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 10.81 +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 10.15 +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 1.57 +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 63-------------------- +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:34:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597465 37.649841 37.551949 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.856934 64.894531 64.744141 +util 95.000000 100.000000 92.000000 +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 46.03 seconds +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 12.09 +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 10.15 +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 1.56 +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 64-------------------- +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:35:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597443 37.649654 37.551960 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.225098 105.244141 105.218750 +util 62.750000 68.000000 57.000000 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 163.37 seconds +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 73.06 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 8.41 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.40 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 65-------------------- +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7285 iter count: 1.0 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7285 iter count: 1.0 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7285 iter count: 1.0 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7285iter count: 1.0 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:38:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597275 37.649010 37.551960 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.637207 64.861328 64.562500 +util 98.250000 100.000000 94.000000 +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 47.13 seconds +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 12.65 +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 10.13 +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 1.57 +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 66-------------------- +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:39:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597282 37.649010 37.551983 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.549316 64.685547 64.503906 +util 97.500000 99.000000 93.000000 +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 45.99 seconds +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 12.07 +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 10.12 +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 1.57 +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 67-------------------- +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:39:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597285 37.649010 37.551960 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.603027 64.841797 64.523438 +util 97.500000 99.000000 93.000000 +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 48.80 seconds +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 13.49 +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 10.11 +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 1.57 +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 68-------------------- +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:40:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597302 37.649055 37.551983 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.588379 64.841797 64.503906 +util 91.750000 99.000000 88.000000 +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 48.67 seconds +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 13.38 +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 10.14 +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 1.58 +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 69-------------------- +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:41:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597298 37.649033 37.551968 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.161621 105.341797 105.101562 +util 68.250000 73.000000 63.000000 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 160.75 seconds +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 71.76 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 8.41 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.40 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 70-------------------- +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7451 iter count: 1.0 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7451 iter count: 1.0 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7451 iter count: 1.0 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7451iter count: 1.0 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:44:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597320 37.649055 37.551991 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.861328 64.878906 64.855469 +util 93.250000 98.000000 87.000000 +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 47.12 seconds +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 12.63 +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 10.14 +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 1.57 +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 71-------------------- +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:45:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597333 37.649033 37.552040 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.788086 64.878906 64.757812 +util 95.500000 100.000000 88.000000 +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 46.09 seconds +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 12.11 +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 10.15 +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 1.57 +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 72-------------------- +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597409 37.649307 37.552040 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.792969 64.898438 64.757812 +util 92.750000 100.000000 89.000000 +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 48.66 seconds +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 13.38 +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 10.16 +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 1.58 +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 73-------------------- +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:46:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597511 37.649578 37.552063 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.792969 64.839844 64.777344 +util 96.750000 100.000000 90.000000 +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 48.56 seconds +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 13.36 +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 10.15 +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 1.56 +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 74-------------------- +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:47:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597569 37.649826 37.552032 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.146973 105.283203 105.101562 +util 65.750000 71.000000 60.000000 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 160.75 seconds +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 71.68 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 8.47 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.40 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 75-------------------- +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6895 iter count: 1.0 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6895 iter count: 1.0 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6895 iter count: 1.0 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6895iter count: 1.0 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:50:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597644 37.650093 37.552032 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.875977 64.937500 64.855469 +util 94.000000 98.000000 90.000000 +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 49.95 seconds +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 14.04 +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 10.14 +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 1.58 +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 76-------------------- +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:50:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597674 37.650093 37.552032 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.836914 64.957031 64.796875 +util 95.250000 99.000000 88.000000 +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 48.69 seconds +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 13.42 +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 10.13 +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 1.58 +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 77-------------------- +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:51:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597679 37.650051 37.552032 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.763672 64.957031 64.699219 +util 95.500000 100.000000 88.000000 +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 48.60 seconds +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 13.38 +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 10.13 +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 1.58 +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 78-------------------- +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:52:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597881 37.650871 37.552036 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734375 64.839844 64.699219 +util 93.250000 99.000000 90.000000 +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 48.66 seconds +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 13.40 +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 10.15 +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 1.57 +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 79-------------------- +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:53:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597659 37.649876 37.552044 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.308105 105.810547 105.140625 +util 67.250000 73.000000 59.000000 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 156.39 seconds +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 69.52 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 8.46 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.42 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 80-------------------- +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7305 iter count: 1.0 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7305 iter count: 1.0 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7305 iter count: 1.0 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7305iter count: 1.0 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:55:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597663 37.649879 37.552052 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817871 64.822266 64.816406 +util 95.250000 100.000000 92.000000 +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 44.65 seconds +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 11.37 +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 10.18 +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 1.57 +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 81-------------------- +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:56:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597665 37.649879 37.552052 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734863 64.738281 64.724609 +util 95.250000 99.000000 90.000000 +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 43.44 seconds +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 10.78 +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 10.13 +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 1.58 +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 82-------------------- +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:57:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597669 37.649879 37.552052 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.715332 64.718750 64.705078 +util 96.750000 100.000000 91.000000 +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 46.09 seconds +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 12.08 +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 10.20 +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 1.57 +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 83-------------------- +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:58:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597867 37.650612 37.552052 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.788574 64.796875 64.763672 +util 92.500000 99.000000 89.000000 +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 46.04 seconds +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 12.08 +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 10.14 +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 1.58 +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 84-------------------- +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 21:58:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597783 37.649921 37.552208 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.239746 105.244141 105.238281 +util 65.250000 70.000000 60.000000 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 161.09 seconds +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 71.92 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 8.44 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.39 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 85-------------------- +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6934 iter count: 1.0 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6934 iter count: 1.0 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6934 iter count: 1.0 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6934iter count: 1.0 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:01:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597793 37.649933 37.552189 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.841797 64.859375 64.835938 +util 91.000000 98.000000 87.000000 +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 47.48 seconds +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 12.83 +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 10.14 +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 1.55 +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 86-------------------- +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:02:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597822 37.649960 37.552189 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.797852 64.800781 64.796875 +util 96.750000 99.000000 91.000000 +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 45.99 seconds +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 12.06 +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 10.14 +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 1.57 +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 87-------------------- +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:03:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597833 37.649979 37.552189 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.797852 64.800781 64.796875 +util 96.750000 99.000000 92.000000 +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 45.94 seconds +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 12.05 +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 10.13 +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 1.57 +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 88-------------------- +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:03:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597825 37.649971 37.552189 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.812500 64.816406 64.800781 +util 95.500000 99.000000 93.000000 +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 46.13 seconds +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 12.11 +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 10.12 +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 1.58 +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 89-------------------- +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:04:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597724 37.649525 37.552193 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.200684 105.380859 105.140625 +util 63.250000 66.000000 62.000000 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 163.41 seconds +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 73.12 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 8.41 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.39 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 90-------------------- +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7354 iter count: 1.0 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7354 iter count: 1.0 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7354 iter count: 1.0 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7354iter count: 1.0 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:07:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597728 37.649521 37.552227 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.837402 64.841797 64.835938 +util 92.250000 99.000000 88.000000 +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 44.55 seconds +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 11.35 +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 10.15 +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 1.57 +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 91-------------------- +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:08:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597852 37.650024 37.552227 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.783691 64.861328 64.757812 +util 93.500000 99.000000 90.000000 +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 48.62 seconds +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 13.37 +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 10.13 +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 1.59 +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 92-------------------- +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:09:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597847 37.650024 37.552227 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.793457 64.841797 64.777344 +util 94.250000 99.000000 87.000000 +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 46.17 seconds +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 12.15 +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 10.14 +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 1.58 +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 93-------------------- +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:09:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597854 37.650024 37.552235 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.773926 64.796875 64.705078 +util 97.500000 100.000000 93.000000 +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 43.33 seconds +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 10.75 +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 10.12 +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 1.57 +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 94-------------------- +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:10:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597624 37.648979 37.552258 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.190918 105.199219 105.166016 +util 64.500000 69.000000 59.000000 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 158.57 seconds +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 70.65 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 8.42 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.40 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 95-------------------- +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7715 iter count: 1.0 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7715 iter count: 1.0 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7715 iter count: 1.0 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7715iter count: 1.0 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:13:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597631 37.648956 37.552258 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.812500 64.917969 64.777344 +util 93.750000 99.000000 90.000000 +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 44.53 seconds +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 11.32 +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 10.15 +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 1.58 +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 96-------------------- +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:13:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597740 37.649204 37.552399 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.724609 64.976562 64.640625 +util 96.000000 100.000000 89.000000 +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 45.93 seconds +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.03 +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 10.14 +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 1.59 +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 97-------------------- +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:14:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597741 37.649204 37.552380 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.753906 64.976562 64.679688 +util 95.250000 99.000000 90.000000 +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 46.13 seconds +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.16 +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 10.13 +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 1.56 +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 98-------------------- +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:15:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597742 37.649227 37.552376 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.744141 64.937500 64.679688 +util 95.250000 99.000000 90.000000 +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 46.00 seconds +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 12.07 +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 10.15 +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 1.57 +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 99-------------------- +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:16:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597628 37.648769 37.552376 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.171387 105.322266 105.121094 +util 67.250000 72.000000 60.000000 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 166.29 seconds +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 74.53 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 8.39 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.40 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 100-------------------- +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7578 iter count: 1.0 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7578 iter count: 1.0 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7578 iter count: 1.0 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7578iter count: 1.0 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 3, 2026 - 22:18:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 3, 2026 - 22:19:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 18.08 seconds +[May 3, 2026 - 22:19:15 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[May 3, 2026 - 22:19:15 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000100 +[May 3, 2026 - 22:20:00 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 100 +[May 3, 2026 - 22:20:00 | INFO | fastgen.trainer:validate:358 ] Validation iteration 100 +[May 3, 2026 - 22:20:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 3, 2026 - 22:20:06 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 3, 2026 - 22:20:07 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.615 +[May 3, 2026 - 22:20:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 3, 2026 - 22:20:12 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.049 +[May 3, 2026 - 22:20:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 3, 2026 - 22:20:17 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.989 +[May 3, 2026 - 22:20:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 3, 2026 - 22:20:21 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.491 +[May 3, 2026 - 22:20:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 3, 2026 - 22:20:26 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.778 +[May 3, 2026 - 22:20:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 3, 2026 - 22:20:31 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.584 +[May 3, 2026 - 22:20:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 3, 2026 - 22:20:36 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.023 +[May 3, 2026 - 22:20:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 3, 2026 - 22:20:41 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.022 +[May 3, 2026 - 22:20:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 3, 2026 - 22:20:46 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.243 +[May 3, 2026 - 22:20:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 3, 2026 - 22:20:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 3, 2026 - 22:20:51 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.584 +[May 3, 2026 - 22:20:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 100-------------------- +[May 3, 2026 - 22:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 3, 2026 - 22:21:08 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 100 +[May 3, 2026 - 22:21:08 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.238 (n=10) +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.436614 41.514168 41.382431 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.094727 68.214844 67.734375 +util 93.250000 98.000000 88.000000 +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 101 : avg iteration time 173.09 seconds +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 101 : data loading time 0.00 +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 101 : avg forward pass time 10.42 +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 101 : backward pass time 10.16 +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 101 : optimizer step time 1.56 +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 101-------------------- +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:21:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.436674 41.514408 41.382431 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.050781 68.175781 67.675781 +util 96.750000 100.000000 91.000000 +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 102 : avg iteration time 48.91 seconds +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 102 : data loading time 0.00 +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 102 : avg forward pass time 13.48 +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 102 : backward pass time 10.18 +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 102 : optimizer step time 1.58 +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 102-------------------- +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.436696 41.514507 41.382412 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.163086 68.175781 68.125000 +util 96.500000 100.000000 90.000000 +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 103 : avg iteration time 43.60 seconds +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 103 : data loading time 0.00 +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 103 : avg forward pass time 10.87 +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 103 : backward pass time 10.12 +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 103 : optimizer step time 1.56 +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 103-------------------- +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:23:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.436180 41.512398 41.382458 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.158203 68.175781 68.105469 +util 97.000000 99.000000 92.000000 +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 104 : avg iteration time 46.35 seconds +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 104 : data loading time 0.00 +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 104 : avg forward pass time 12.22 +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 104 : backward pass time 10.15 +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 104 : optimizer step time 1.57 +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 104-------------------- +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:24:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437095 41.512051 41.383675 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.628906 107.544922 +util 69.000000 74.000000 64.000000 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 105 : avg iteration time 161.24 seconds +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 105 : data loading time 0.00 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 105 : avg forward pass time 71.95 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 105 : backward pass time 8.48 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 105 : optimizer step time 0.40 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 105-------------------- +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7520 iter count: 1.0 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7520 iter count: 1.0 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7520 iter count: 1.0 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7520iter count: 1.0 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:26:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437102 41.512051 41.383675 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.286133 69.894531 67.460938 +util 95.250000 99.000000 89.000000 +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 106 : avg iteration time 44.58 seconds +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 106 : data loading time 0.00 +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 106 : avg forward pass time 11.37 +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 106 : backward pass time 10.13 +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 106 : optimizer step time 1.57 +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 106-------------------- +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:27:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437228 41.512550 41.383678 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.334961 69.992188 67.363281 +util 93.000000 99.000000 89.000000 +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 107 : avg iteration time 48.87 seconds +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 107 : data loading time 0.00 +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 107 : avg forward pass time 13.49 +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 107 : backward pass time 10.15 +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 107 : optimizer step time 1.58 +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 107-------------------- +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:28:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437234 41.512573 41.383678 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.393555 70.070312 67.363281 +util 93.000000 99.000000 89.000000 +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 108 : avg iteration time 48.61 seconds +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 108 : data loading time 0.00 +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 108 : avg forward pass time 13.37 +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 108 : backward pass time 10.14 +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 108 : optimizer step time 1.57 +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 108-------------------- +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:29:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437254 41.512608 41.383682 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.369141 70.031250 67.382812 +util 91.500000 99.000000 87.000000 +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 109 : avg iteration time 46.18 seconds +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 109 : data loading time 0.00 +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 109 : avg forward pass time 12.15 +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 109 : backward pass time 10.14 +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 109 : optimizer step time 1.57 +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 109-------------------- +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:29:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437105 41.511776 41.383812 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.667969 107.564453 +util 64.750000 72.000000 60.000000 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 110 : avg iteration time 161.80 seconds +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 110 : data loading time 0.00 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 110 : avg forward pass time 72.23 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 110 : backward pass time 8.48 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 110 : optimizer step time 0.41 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 110-------------------- +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7441 iter count: 1.0 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7441 iter count: 1.0 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7441 iter count: 1.0 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7441iter count: 1.0 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:32:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437110 41.511734 41.383812 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.129395 67.382812 67.044922 +util 95.500000 99.000000 88.000000 +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 111 : avg iteration time 47.56 seconds +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 111 : data loading time 0.00 +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 111 : avg forward pass time 12.84 +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 111 : backward pass time 10.16 +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 111 : optimizer step time 1.56 +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 111-------------------- +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:33:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437182 41.511986 41.383812 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.329590 67.363281 67.318359 +util 95.500000 99.000000 91.000000 +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 112 : avg iteration time 46.15 seconds +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 112 : data loading time 0.00 +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 112 : avg forward pass time 12.13 +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 112 : backward pass time 10.16 +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 112 : optimizer step time 1.58 +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 112-------------------- +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:34:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437265 41.512245 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.339355 67.402344 67.318359 +util 96.250000 100.000000 90.000000 +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 113 : avg iteration time 43.47 seconds +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 113 : data loading time 0.00 +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 113 : avg forward pass time 10.79 +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 113 : backward pass time 10.14 +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 113 : optimizer step time 1.58 +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 113-------------------- +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:34:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437295 41.512245 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.334473 67.382812 67.318359 +util 93.750000 99.000000 90.000000 +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 114 : avg iteration time 43.46 seconds +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 114 : data loading time 0.00 +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 114 : avg forward pass time 10.80 +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 114 : backward pass time 10.14 +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 114 : optimizer step time 1.59 +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 114-------------------- +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:35:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437072 41.511333 41.383823 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.536621 107.628906 107.505859 +util 66.250000 71.000000 61.000000 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 115 : avg iteration time 163.69 seconds +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 115 : data loading time 0.00 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 115 : avg forward pass time 73.27 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 115 : backward pass time 8.39 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 115 : optimizer step time 0.40 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 115-------------------- +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7148 iter count: 1.0 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7148 iter count: 1.0 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7148 iter count: 1.0 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7148iter count: 1.0 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:38:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437121 41.511509 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.123047 67.363281 67.042969 +util 92.250000 99.000000 88.000000 +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 116 : avg iteration time 50.09 seconds +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 116 : data loading time 0.00 +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 116 : avg forward pass time 14.09 +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 116 : backward pass time 10.16 +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 116 : optimizer step time 1.59 +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 116-------------------- +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:39:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437640 41.513584 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.441406 67.316406 +util 95.250000 100.000000 91.000000 +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 117 : avg iteration time 43.43 seconds +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 117 : data loading time 0.00 +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 117 : avg forward pass time 10.78 +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 117 : backward pass time 10.15 +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 117 : optimizer step time 1.58 +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 117-------------------- +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:39:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437643 41.513584 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.157227 67.500000 67.042969 +util 95.750000 99.000000 91.000000 +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 118 : avg iteration time 43.46 seconds +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 118 : data loading time 0.00 +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 118 : avg forward pass time 10.81 +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 118 : backward pass time 10.13 +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 118 : optimizer step time 1.56 +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 118-------------------- +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437654 41.513649 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.376953 67.382812 67.375000 +util 94.000000 100.000000 90.000000 +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 119 : avg iteration time 46.13 seconds +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 119 : data loading time 0.00 +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 119 : avg forward pass time 12.11 +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 119 : backward pass time 10.13 +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 119 : optimizer step time 1.57 +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 119-------------------- +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437283 41.512142 41.383835 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.628906 107.623047 +util 66.750000 72.000000 59.000000 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 120 : avg iteration time 163.55 seconds +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 120 : data loading time 0.00 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 120 : avg forward pass time 73.13 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 120 : backward pass time 8.40 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 120 : optimizer step time 0.40 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 120-------------------- +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7764 iter count: 1.0 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7764 iter count: 1.0 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7764 iter count: 1.0 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7764iter count: 1.0 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:44:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437287 41.512127 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.127930 67.441406 67.023438 +util 96.000000 100.000000 91.000000 +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 121 : avg iteration time 44.72 seconds +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 121 : data loading time 0.00 +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 121 : avg forward pass time 11.43 +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 121 : backward pass time 10.14 +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 121 : optimizer step time 1.58 +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 121-------------------- +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:44:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437387 41.512455 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.210938 67.363281 67.160156 +util 93.250000 99.000000 89.000000 +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 122 : avg iteration time 46.13 seconds +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 122 : data loading time 0.00 +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 122 : avg forward pass time 12.13 +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 122 : backward pass time 10.15 +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 122 : optimizer step time 1.57 +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 122-------------------- +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:45:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437768 41.513935 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.210938 67.363281 67.160156 +util 93.000000 98.000000 88.000000 +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 123 : avg iteration time 46.24 seconds +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 123 : data loading time 0.00 +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 123 : avg forward pass time 12.18 +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 123 : backward pass time 10.15 +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 123 : optimizer step time 1.58 +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 123-------------------- +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:46:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437820 41.514145 41.383823 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.210938 67.363281 67.160156 +util 92.500000 100.000000 88.000000 +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 124 : avg iteration time 46.15 seconds +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 124 : data loading time 0.00 +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 124 : avg forward pass time 12.16 +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 124 : backward pass time 10.14 +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 124 : optimizer step time 1.56 +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 124-------------------- +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:47:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437318 41.512074 41.383926 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.648438 107.642578 +util 67.500000 72.000000 63.000000 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 125 : avg iteration time 158.33 seconds +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 125 : data loading time 0.00 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 125 : avg forward pass time 70.58 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 125 : backward pass time 8.43 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 125 : optimizer step time 0.40 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 125-------------------- +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6172 iter count: 1.0 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6172 iter count: 1.0 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6172 iter count: 1.0 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6172iter count: 1.0 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:49:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437354 41.512100 41.383995 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.363770 67.382812 67.357422 +util 95.500000 100.000000 88.000000 +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 126 : avg iteration time 49.81 seconds +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 126 : data loading time 0.00 +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 126 : avg forward pass time 13.98 +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 126 : backward pass time 10.14 +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 126 : optimizer step time 1.57 +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 126-------------------- +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437435 41.512482 41.383980 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.246582 67.382812 67.201172 +util 98.250000 100.000000 94.000000 +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 127 : avg iteration time 43.39 seconds +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 127 : data loading time 0.00 +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 127 : avg forward pass time 10.77 +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 127 : backward pass time 10.13 +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 127 : optimizer step time 1.58 +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 127-------------------- +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437628 41.513187 41.383980 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.378418 67.382812 67.376953 +util 95.250000 100.000000 90.000000 +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 128 : avg iteration time 43.36 seconds +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 128 : data loading time 0.00 +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 128 : avg forward pass time 10.76 +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 128 : backward pass time 10.12 +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 128 : optimizer step time 1.57 +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 128-------------------- +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:52:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437632 41.513187 41.383995 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.100098 67.382812 67.005859 +util 95.250000 100.000000 91.000000 +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 129 : avg iteration time 43.56 seconds +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 129 : data loading time 0.00 +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 129 : avg forward pass time 10.85 +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 129 : backward pass time 10.14 +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 129 : optimizer step time 1.57 +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 129-------------------- +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:52:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437236 41.511581 41.383991 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.687500 107.623047 +util 63.250000 68.000000 57.000000 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 130 : avg iteration time 161.04 seconds +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 130 : data loading time 0.00 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 130 : avg forward pass time 71.89 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 130 : backward pass time 8.41 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 130 : optimizer step time 0.40 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 130-------------------- +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7002 iter count: 1.0 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7002 iter count: 1.0 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7002 iter count: 1.0 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7002iter count: 1.0 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:55:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437284 41.511581 41.384182 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.148926 67.460938 67.044922 +util 95.250000 99.000000 88.000000 +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 131 : avg iteration time 50.06 seconds +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 131 : data loading time 0.00 +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 131 : avg forward pass time 14.09 +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 131 : backward pass time 10.15 +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 131 : optimizer step time 1.58 +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 131-------------------- +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:56:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437358 41.511883 41.384182 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.251465 67.460938 67.181641 +util 95.750000 99.000000 89.000000 +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 132 : avg iteration time 46.22 seconds +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 132 : data loading time 0.00 +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 132 : avg forward pass time 12.17 +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 132 : backward pass time 10.15 +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 132 : optimizer step time 1.56 +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 132-------------------- +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:57:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437601 41.512859 41.384193 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.275879 67.500000 67.201172 +util 97.250000 99.000000 93.000000 +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 133 : avg iteration time 46.18 seconds +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 133 : data loading time 0.00 +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 133 : avg forward pass time 12.14 +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 133 : backward pass time 10.16 +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 133 : optimizer step time 1.58 +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 133-------------------- +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:57:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437652 41.513073 41.384182 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.241699 67.363281 67.201172 +util 95.750000 99.000000 89.000000 +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 134 : avg iteration time 48.82 seconds +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 134 : data loading time 0.00 +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 134 : avg forward pass time 13.46 +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 134 : backward pass time 10.17 +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 134 : optimizer step time 1.57 +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 134-------------------- +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 22:58:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437611 41.512783 41.384205 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.536621 107.687500 107.486328 +util 64.250000 70.000000 60.000000 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 135 : avg iteration time 158.53 seconds +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 135 : data loading time 0.00 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 135 : avg forward pass time 70.66 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 135 : backward pass time 8.40 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 135 : optimizer step time 0.40 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 135-------------------- +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6865 iter count: 1.0 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6865 iter count: 1.0 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6865 iter count: 1.0 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6865iter count: 1.0 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:01:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437697 41.513130 41.384205 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.767578 67.480469 +util 93.250000 98.000000 88.000000 +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 136 : avg iteration time 47.39 seconds +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 136 : data loading time 0.00 +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 136 : avg forward pass time 12.75 +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 136 : backward pass time 10.13 +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 136 : optimizer step time 1.59 +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 136-------------------- +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:02:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437689 41.513111 41.384182 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.637207 67.669922 67.539062 +util 96.750000 99.000000 91.000000 +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 137 : avg iteration time 46.11 seconds +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 137 : data loading time 0.00 +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 137 : avg forward pass time 12.12 +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 137 : backward pass time 10.16 +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 137 : optimizer step time 1.56 +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 137-------------------- +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437766 41.513390 41.384205 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.632324 67.669922 67.519531 +util 95.500000 99.000000 88.000000 +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 138 : avg iteration time 46.17 seconds +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 138 : data loading time 0.00 +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 138 : avg forward pass time 12.14 +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 138 : backward pass time 10.14 +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 138 : optimizer step time 1.58 +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 138-------------------- +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:03:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437785 41.513355 41.384205 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.637207 67.669922 67.539062 +util 97.500000 99.000000 94.000000 +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 139 : avg iteration time 45.98 seconds +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 139 : data loading time 0.00 +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 139 : avg forward pass time 12.06 +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 139 : backward pass time 10.14 +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 139 : optimizer step time 1.57 +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 139-------------------- +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437404 41.511837 41.384182 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.687500 107.544922 +util 62.250000 70.000000 57.000000 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 140 : avg iteration time 163.70 seconds +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 140 : data loading time 0.00 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 140 : avg forward pass time 73.23 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 140 : backward pass time 8.42 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 140 : optimizer step time 0.40 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 140-------------------- +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7295 iter count: 1.0 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7295 iter count: 1.0 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7295 iter count: 1.0 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7295iter count: 1.0 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:07:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437417 41.511837 41.384228 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.500488 67.552734 67.343750 +util 94.750000 99.000000 90.000000 +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 141 : avg iteration time 47.66 seconds +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 141 : data loading time 0.00 +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 141 : avg forward pass time 12.88 +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 141 : backward pass time 10.14 +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 141 : optimizer step time 1.58 +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 141-------------------- +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:08:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437778 41.513306 41.384186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.656738 67.748047 67.382812 +util 95.250000 99.000000 88.000000 +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 142 : avg iteration time 46.06 seconds +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 142 : data loading time 0.00 +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 142 : avg forward pass time 12.11 +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 142 : backward pass time 10.13 +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 142 : optimizer step time 1.56 +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 142-------------------- +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:08:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437898 41.513779 41.384209 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358887 67.363281 67.357422 +util 96.750000 100.000000 92.000000 +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 143 : avg iteration time 46.16 seconds +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 143 : data loading time 0.00 +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 143 : avg forward pass time 12.15 +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 143 : backward pass time 10.14 +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 143 : optimizer step time 1.57 +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 143-------------------- +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:09:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437975 41.514061 41.384209 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.563965 67.630859 67.363281 +util 94.000000 100.000000 90.000000 +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 144 : avg iteration time 43.52 seconds +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 144 : data loading time 0.00 +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 144 : avg forward pass time 10.83 +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 144 : backward pass time 10.14 +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 144 : optimizer step time 1.58 +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 144-------------------- +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:10:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437594 41.512562 41.384186 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.765625 107.603516 +util 66.250000 72.000000 57.000000 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 145 : avg iteration time 161.37 seconds +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 145 : data loading time 0.00 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 145 : avg forward pass time 72.07 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 145 : backward pass time 8.44 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 145 : optimizer step time 0.43 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 145-------------------- +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7109 iter count: 1.0 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7109 iter count: 1.0 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7109 iter count: 1.0 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7109iter count: 1.0 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:12:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437616 41.512585 41.384209 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.635742 67.707031 67.421875 +util 96.500000 99.000000 90.000000 +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 146 : avg iteration time 47.17 seconds +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 146 : data loading time 0.00 +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 146 : avg forward pass time 12.66 +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 146 : backward pass time 10.16 +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 146 : optimizer step time 1.57 +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 146-------------------- +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:13:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437779 41.513283 41.384186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.293945 67.460938 67.238281 +util 95.750000 100.000000 89.000000 +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 147 : avg iteration time 48.78 seconds +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 147 : data loading time 0.00 +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 147 : avg forward pass time 13.44 +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 147 : backward pass time 10.14 +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 147 : optimizer step time 1.57 +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 147-------------------- +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:14:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437980 41.514061 41.384186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.313477 67.480469 67.257812 +util 95.750000 100.000000 91.000000 +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 148 : avg iteration time 46.15 seconds +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 148 : data loading time 0.00 +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 148 : avg forward pass time 12.14 +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 148 : backward pass time 10.16 +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 148 : optimizer step time 1.57 +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 148-------------------- +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:15:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437977 41.514015 41.384186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396484 67.460938 67.375000 +util 96.500000 99.000000 93.000000 +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 149 : avg iteration time 43.62 seconds +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 149 : data loading time 0.00 +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 149 : avg forward pass time 10.87 +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 149 : backward pass time 10.14 +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 149 : optimizer step time 1.59 +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 149-------------------- +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:16:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437449 41.511948 41.384186 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.824707 108.371094 107.642578 +util 68.000000 73.000000 60.000000 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 150 : avg iteration time 161.45 seconds +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 150 : data loading time 0.00 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 150 : avg forward pass time 72.09 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 150 : backward pass time 8.45 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 150 : optimizer step time 0.40 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 150-------------------- +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6826 iter count: 1.0 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6826 iter count: 1.0 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6826 iter count: 1.0 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6826iter count: 1.0 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:18:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437445 41.511925 41.384190 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.627441 67.708984 67.382812 +util 97.000000 99.000000 91.000000 +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 151 : avg iteration time 47.19 seconds +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 151 : data loading time 0.00 +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 151 : avg forward pass time 12.65 +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 151 : backward pass time 10.13 +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 151 : optimizer step time 1.59 +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 151-------------------- +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:19:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437564 41.512405 41.384186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.632324 67.689453 67.460938 +util 92.000000 99.000000 87.000000 +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 152 : avg iteration time 48.82 seconds +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 152 : data loading time 0.00 +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 152 : avg forward pass time 13.49 +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 152 : backward pass time 10.10 +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 152 : optimizer step time 1.56 +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 152-------------------- +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:20:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437939 41.513889 41.384186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.622559 67.708984 67.363281 +util 93.250000 99.000000 89.000000 +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 153 : avg iteration time 46.22 seconds +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 153 : data loading time 0.00 +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 153 : avg forward pass time 12.17 +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 153 : backward pass time 10.16 +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 153 : optimizer step time 1.56 +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 153-------------------- +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:21:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437992 41.514114 41.384186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.500488 67.533203 67.402344 +util 92.500000 97.000000 89.000000 +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 154 : avg iteration time 43.63 seconds +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 154 : data loading time 0.00 +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 154 : avg forward pass time 10.87 +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 154 : backward pass time 10.15 +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 154 : optimizer step time 1.57 +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 154-------------------- +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437481 41.512001 41.384186 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.667969 107.583984 +util 61.250000 70.000000 55.000000 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 155 : avg iteration time 158.98 seconds +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 155 : data loading time 0.00 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 155 : avg forward pass time 70.87 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 155 : backward pass time 8.43 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 155 : optimizer step time 0.43 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 155-------------------- +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6787 iter count: 1.0 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6787 iter count: 1.0 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6787 iter count: 1.0 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6787iter count: 1.0 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:24:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437490 41.512001 41.384212 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.266113 67.460938 67.201172 +util 95.250000 99.000000 91.000000 +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 156 : avg iteration time 49.85 seconds +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 156 : data loading time 0.00 +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 156 : avg forward pass time 13.97 +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 156 : backward pass time 10.15 +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 156 : optimizer step time 1.57 +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 156-------------------- +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:25:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437608 41.512489 41.384220 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.095215 67.421875 66.986328 +util 94.750000 99.000000 91.000000 +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 157 : avg iteration time 46.15 seconds +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 157 : data loading time 0.00 +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 157 : avg forward pass time 12.14 +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 157 : backward pass time 10.15 +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 157 : optimizer step time 1.57 +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 157-------------------- +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:26:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437927 41.513718 41.384220 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.197754 67.421875 67.123047 +util 94.250000 99.000000 90.000000 +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 158 : avg iteration time 46.20 seconds +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 158 : data loading time 0.00 +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 158 : avg forward pass time 12.17 +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 158 : backward pass time 10.14 +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 158 : optimizer step time 1.57 +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 158-------------------- +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:26:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437969 41.513721 41.384247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.285645 67.421875 67.240234 +util 92.000000 97.000000 87.000000 +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 159 : avg iteration time 46.20 seconds +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 159 : data loading time 0.00 +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 159 : avg forward pass time 12.16 +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 159 : backward pass time 10.16 +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 159 : optimizer step time 1.57 +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 159-------------------- +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:27:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437586 41.512180 41.384247 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.667969 107.564453 +util 64.750000 69.000000 61.000000 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 160 : avg iteration time 163.65 seconds +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 160 : data loading time 0.00 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 160 : avg forward pass time 73.23 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 160 : backward pass time 8.41 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 160 : optimizer step time 0.40 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 160-------------------- +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7002 iter count: 1.0 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7002 iter count: 1.0 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7002 iter count: 1.0 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7002iter count: 1.0 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:30:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437603 41.512180 41.384354 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.264648 67.460938 67.199219 +util 93.000000 98.000000 88.000000 +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 161 : avg iteration time 44.76 seconds +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 161 : data loading time 0.00 +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 161 : avg forward pass time 11.45 +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 161 : backward pass time 10.14 +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 161 : optimizer step time 1.58 +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 161-------------------- +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:31:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437797 41.512936 41.384354 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.123047 67.363281 67.042969 +util 92.250000 99.000000 87.000000 +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 162 : avg iteration time 46.14 seconds +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 162 : data loading time 0.00 +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 162 : avg forward pass time 12.14 +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 162 : backward pass time 10.14 +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 162 : optimizer step time 1.57 +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 162-------------------- +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:31:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438115 41.514149 41.384354 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.240234 67.363281 67.199219 +util 96.250000 100.000000 89.000000 +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 163 : avg iteration time 46.15 seconds +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 163 : data loading time 0.00 +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 163 : avg forward pass time 12.16 +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 163 : backward pass time 10.14 +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 163 : optimizer step time 1.56 +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 163-------------------- +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:32:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438101 41.514149 41.384354 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.225586 67.363281 67.179688 +util 97.000000 99.000000 91.000000 +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 164 : avg iteration time 46.42 seconds +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 164 : data loading time 0.00 +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 164 : avg forward pass time 12.27 +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 164 : backward pass time 10.14 +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 164 : optimizer step time 1.58 +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 164-------------------- +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:33:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437574 41.512032 41.384354 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.628906 107.544922 +util 63.750000 71.000000 58.000000 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 165 : avg iteration time 160.85 seconds +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 165 : data loading time 0.00 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 165 : avg forward pass time 71.85 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 165 : backward pass time 8.40 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 165 : optimizer step time 0.39 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 165-------------------- +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6943 iter count: 1.0 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6943 iter count: 1.0 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6943 iter count: 1.0 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6943iter count: 1.0 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:36:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437592 41.512054 41.384354 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.461426 67.494141 67.363281 +util 97.250000 100.000000 93.000000 +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 166 : avg iteration time 50.09 seconds +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 166 : data loading time 0.00 +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 166 : avg forward pass time 14.11 +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 166 : backward pass time 10.15 +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 166 : optimizer step time 1.57 +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 166-------------------- +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:36:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437694 41.512531 41.384354 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.388184 67.480469 67.357422 +util 94.000000 99.000000 89.000000 +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 167 : avg iteration time 48.75 seconds +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 167 : data loading time 0.00 +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 167 : avg forward pass time 13.45 +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 167 : backward pass time 10.15 +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 167 : optimizer step time 1.58 +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 167-------------------- +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:37:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437957 41.513557 41.384369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.441895 67.474609 67.343750 +util 97.250000 100.000000 91.000000 +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 168 : avg iteration time 46.11 seconds +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 168 : data loading time 0.00 +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 168 : avg forward pass time 12.12 +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 168 : backward pass time 10.14 +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 168 : optimizer step time 1.58 +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 168-------------------- +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:38:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437969 41.513523 41.384373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.627441 67.669922 67.500000 +util 92.750000 99.000000 88.000000 +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 169 : avg iteration time 46.23 seconds +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 169 : data loading time 0.00 +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 169 : avg forward pass time 12.19 +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 169 : backward pass time 10.16 +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 169 : optimizer step time 1.56 +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 169-------------------- +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:39:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437521 41.511700 41.384373 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.804688 107.486328 +util 64.250000 74.000000 54.000000 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 170 : avg iteration time 161.18 seconds +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 170 : data loading time 0.00 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 170 : avg forward pass time 72.00 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 170 : backward pass time 8.43 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 170 : optimizer step time 0.40 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 170-------------------- +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6982 iter count: 1.0 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6982 iter count: 1.0 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6982 iter count: 1.0 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6982iter count: 1.0 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:41:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437528 41.511700 41.384399 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373535 67.480469 67.337891 +util 91.750000 99.000000 87.000000 +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 171 : avg iteration time 47.40 seconds +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 171 : data loading time 0.00 +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 171 : avg forward pass time 12.75 +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 171 : backward pass time 10.15 +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 171 : optimizer step time 1.58 +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 171-------------------- +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:42:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437567 41.511879 41.384354 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.383301 67.460938 67.357422 +util 94.250000 99.000000 90.000000 +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 172 : avg iteration time 46.08 seconds +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 172 : data loading time 0.00 +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 172 : avg forward pass time 12.09 +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 172 : backward pass time 10.14 +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 172 : optimizer step time 1.58 +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 172-------------------- +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:43:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437869 41.513107 41.384357 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.280762 67.460938 67.220703 +util 94.250000 100.000000 90.000000 +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 173 : avg iteration time 46.36 seconds +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 173 : data loading time 0.00 +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 173 : avg forward pass time 12.24 +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 173 : backward pass time 10.14 +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 173 : optimizer step time 1.58 +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 173-------------------- +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:44:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437876 41.513107 41.384384 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.368652 67.460938 67.337891 +util 95.000000 99.000000 91.000000 +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 174 : avg iteration time 46.25 seconds +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 174 : data loading time 0.00 +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 174 : avg forward pass time 12.16 +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 174 : backward pass time 10.14 +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 174 : optimizer step time 1.57 +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 174-------------------- +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:45:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437924 41.513260 41.384403 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.561035 107.609375 107.544922 +util 66.250000 71.000000 59.000000 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 175 : avg iteration time 158.68 seconds +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 175 : data loading time 0.00 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 175 : avg forward pass time 70.74 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 175 : backward pass time 8.39 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 175 : optimizer step time 0.40 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 175-------------------- +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6846 iter count: 1.0 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6846 iter count: 1.0 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6846 iter count: 1.0 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6846iter count: 1.0 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:47:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437803 41.512783 41.384418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.264648 67.460938 67.199219 +util 97.750000 99.000000 95.000000 +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 176 : avg iteration time 47.37 seconds +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 176 : data loading time 0.00 +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 176 : avg forward pass time 12.76 +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 176 : backward pass time 10.14 +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 176 : optimizer step time 1.56 +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 176-------------------- +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:48:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437801 41.512775 41.384418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.240234 67.363281 67.199219 +util 92.750000 99.000000 88.000000 +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 177 : avg iteration time 46.28 seconds +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 177 : data loading time 0.00 +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 177 : avg forward pass time 12.19 +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 177 : backward pass time 10.16 +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 177 : optimizer step time 1.58 +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 177-------------------- +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:49:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437876 41.513035 41.384441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.196289 67.363281 67.140625 +util 94.000000 99.000000 90.000000 +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 178 : avg iteration time 43.62 seconds +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 178 : data loading time 0.00 +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 178 : avg forward pass time 10.86 +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 178 : backward pass time 10.15 +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 178 : optimizer step time 1.59 +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 178-------------------- +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:50:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437871 41.513035 41.384418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.181641 67.363281 67.121094 +util 96.000000 99.000000 89.000000 +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 179 : avg iteration time 48.72 seconds +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 179 : data loading time 0.00 +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 179 : avg forward pass time 13.42 +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 179 : backward pass time 10.16 +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 179 : optimizer step time 1.56 +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 179-------------------- +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:50:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437885 41.513035 41.384418 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.536621 107.687500 107.486328 +util 64.000000 72.000000 58.000000 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 180 : avg iteration time 163.89 seconds +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 180 : data loading time 0.00 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 180 : avg forward pass time 73.33 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 180 : backward pass time 8.42 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 180 : optimizer step time 0.40 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 180-------------------- +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7373 iter count: 1.0 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7373 iter count: 1.0 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7373 iter count: 1.0 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7373iter count: 1.0 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:53:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437906 41.512985 41.384441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.500000 67.376953 +util 98.000000 100.000000 95.000000 +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 181 : avg iteration time 47.29 seconds +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 181 : data loading time 0.00 +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 181 : avg forward pass time 12.70 +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 181 : backward pass time 10.15 +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 181 : optimizer step time 1.57 +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 181-------------------- +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:54:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437900 41.512985 41.384418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.524902 67.552734 67.441406 +util 96.500000 99.000000 93.000000 +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 182 : avg iteration time 46.17 seconds +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 182 : data loading time 0.00 +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 182 : avg forward pass time 12.16 +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 182 : backward pass time 10.12 +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 182 : optimizer step time 1.58 +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 182-------------------- +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:55:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437966 41.513226 41.384441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.427246 67.519531 67.396484 +util 96.500000 99.000000 90.000000 +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 183 : avg iteration time 48.67 seconds +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 183 : data loading time 0.00 +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 183 : avg forward pass time 13.38 +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 183 : backward pass time 10.18 +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 183 : optimizer step time 1.58 +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 183-------------------- +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:55:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437957 41.513229 41.384418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.432129 67.539062 67.396484 +util 92.000000 100.000000 86.000000 +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 184 : avg iteration time 43.43 seconds +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 184 : data loading time 0.00 +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 184 : avg forward pass time 10.78 +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 184 : backward pass time 10.15 +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 184 : optimizer step time 1.57 +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 184-------------------- +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:56:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437970 41.513229 41.384460 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.804688 107.564453 +util 64.000000 69.000000 59.000000 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 185 : avg iteration time 158.29 seconds +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 185 : data loading time 0.00 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 185 : avg forward pass time 70.50 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 185 : backward pass time 8.40 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 185 : optimizer step time 0.40 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 185-------------------- +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6504 iter count: 1.0 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6504 iter count: 1.0 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6504 iter count: 1.0 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6504iter count: 1.0 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 3, 2026 - 23:59:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437909 41.512936 41.384483 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.503906 67.550781 67.363281 +util 92.250000 99.000000 87.000000 +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 186 : avg iteration time 49.92 seconds +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 186 : data loading time 0.00 +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 186 : avg forward pass time 14.03 +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 186 : backward pass time 10.14 +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 186 : optimizer step time 1.58 +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 186-------------------- +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:00:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437908 41.512936 41.384480 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.054688 67.382812 66.945312 +util 95.500000 100.000000 88.000000 +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 187 : avg iteration time 43.53 seconds +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 187 : data loading time 0.00 +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 187 : avg forward pass time 10.84 +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 187 : backward pass time 10.13 +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 187 : optimizer step time 1.57 +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 187-------------------- +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:00:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437903 41.512936 41.384460 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.313477 67.597656 67.218750 +util 95.000000 100.000000 89.000000 +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 188 : avg iteration time 43.45 seconds +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 188 : data loading time 0.00 +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 188 : avg forward pass time 10.81 +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 188 : backward pass time 10.13 +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 188 : optimizer step time 1.58 +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 188-------------------- +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:01:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438029 41.513428 41.384464 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.474609 67.597656 67.433594 +util 93.000000 100.000000 88.000000 +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 189 : avg iteration time 48.62 seconds +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 189 : data loading time 0.00 +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 189 : avg forward pass time 13.38 +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 189 : backward pass time 10.15 +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 189 : optimizer step time 1.57 +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 189-------------------- +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:02:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437674 41.511967 41.384487 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.536621 107.628906 107.505859 +util 66.250000 72.000000 58.000000 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 190 : avg iteration time 163.55 seconds +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 190 : data loading time 0.00 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 190 : avg forward pass time 73.12 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 190 : backward pass time 8.40 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 190 : optimizer step time 0.40 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 190-------------------- +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6621 iter count: 1.0 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6621 iter count: 1.0 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6621 iter count: 1.0 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6621iter count: 1.0 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:05:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437670 41.511971 41.384464 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.480469 67.376953 +util 95.000000 100.000000 89.000000 +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 191 : avg iteration time 49.82 seconds +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 191 : data loading time 0.00 +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 191 : avg forward pass time 13.99 +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 191 : backward pass time 10.13 +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 191 : optimizer step time 1.58 +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 191-------------------- +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:05:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437704 41.512051 41.384464 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.494141 67.441406 +util 97.000000 99.000000 91.000000 +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 192 : avg iteration time 43.38 seconds +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 192 : data loading time 0.00 +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 192 : avg forward pass time 10.75 +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 192 : backward pass time 10.15 +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 192 : optimizer step time 1.58 +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 192-------------------- +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:06:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437711 41.512074 41.384468 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.495605 67.513672 67.441406 +util 96.000000 99.000000 92.000000 +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 193 : avg iteration time 48.93 seconds +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 193 : data loading time 0.00 +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 193 : avg forward pass time 13.53 +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 193 : backward pass time 10.14 +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 193 : optimizer step time 1.57 +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 193-------------------- +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:07:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438014 41.513298 41.384464 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.378418 67.441406 67.357422 +util 97.250000 100.000000 91.000000 +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 194 : avg iteration time 48.68 seconds +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 194 : data loading time 0.00 +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 194 : avg forward pass time 13.41 +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 194 : backward pass time 10.14 +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 194 : optimizer step time 1.57 +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 194-------------------- +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:08:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437719 41.512093 41.384472 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.512207 107.589844 107.486328 +util 65.500000 74.000000 59.000000 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 195 : avg iteration time 156.08 seconds +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 195 : data loading time 0.00 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 195 : avg forward pass time 69.46 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 195 : backward pass time 8.41 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 195 : optimizer step time 0.40 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 195-------------------- +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6084 iter count: 1.0 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6084 iter count: 1.0 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6084 iter count: 1.0 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6084iter count: 1.0 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:10:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437742 41.512115 41.384464 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.500000 67.376953 +util 95.000000 100.000000 89.000000 +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 196 : avg iteration time 49.93 seconds +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 196 : data loading time 0.00 +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 196 : avg forward pass time 14.04 +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 196 : backward pass time 10.14 +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 196 : optimizer step time 1.58 +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 196-------------------- +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:11:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437861 41.512566 41.384464 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.397949 67.402344 67.396484 +util 94.750000 99.000000 89.000000 +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 197 : avg iteration time 48.79 seconds +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 197 : data loading time 0.00 +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 197 : avg forward pass time 13.46 +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 197 : backward pass time 10.14 +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 197 : optimizer step time 1.57 +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 197-------------------- +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:12:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437879 41.512569 41.384464 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.378418 67.382812 67.376953 +util 97.500000 99.000000 95.000000 +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 198 : avg iteration time 46.14 seconds +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 198 : data loading time 0.00 +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 198 : avg forward pass time 12.14 +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 198 : backward pass time 10.15 +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 198 : optimizer step time 1.58 +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 198-------------------- +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:13:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437891 41.512569 41.384464 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.505371 67.533203 67.421875 +util 95.000000 99.000000 91.000000 +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 199 : avg iteration time 46.15 seconds +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 199 : data loading time 0.00 +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 199 : avg forward pass time 12.15 +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 199 : backward pass time 10.14 +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 199 : optimizer step time 1.57 +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 199-------------------- +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:14:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.437856 41.512409 41.384483 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.561035 107.609375 107.544922 +util 67.750000 73.000000 60.000000 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 200 : avg iteration time 163.91 seconds +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 200 : data loading time 0.00 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 200 : avg forward pass time 73.34 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 200 : backward pass time 8.43 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 200 : optimizer step time 0.40 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 200-------------------- +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6719 iter count: 1.0 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6719 iter count: 1.0 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6719 iter count: 1.0 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6719iter count: 1.0 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 00:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 00:17:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.33 seconds +[May 4, 2026 - 00:17:03 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 200 +[May 4, 2026 - 00:17:03 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000200 +[May 4, 2026 - 00:17:42 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 200 +[May 4, 2026 - 00:17:42 | INFO | fastgen.trainer:validate:358 ] Validation iteration 200 +[May 4, 2026 - 00:17:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:17:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 4, 2026 - 00:17:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.398 +[May 4, 2026 - 00:17:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:17:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 00:17:54 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.951 +[May 4, 2026 - 00:17:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:17:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 00:17:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.022 +[May 4, 2026 - 00:17:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:18:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 00:18:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.106 +[May 4, 2026 - 00:18:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:18:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 00:18:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.444 +[May 4, 2026 - 00:18:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:18:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 00:18:14 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.232 +[May 4, 2026 - 00:18:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:18:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 00:18:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.873 +[May 4, 2026 - 00:18:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:18:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 00:18:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.016 +[May 4, 2026 - 00:18:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:18:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 00:18:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.972 +[May 4, 2026 - 00:18:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 00:18:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 00:18:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.000 +[May 4, 2026 - 00:18:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 200-------------------- +[May 4, 2026 - 00:18:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 00:18:50 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 200 +[May 4, 2026 - 00:18:50 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.501 (n=10) +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440229 41.520565 41.384872 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.943359 68.886719 67.628906 +util 96.500000 99.000000 90.000000 +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 201 : avg iteration time 166.78 seconds +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 201 : data loading time 0.00 +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 201 : avg forward pass time 11.78 +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 201 : backward pass time 10.14 +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 201 : optimizer step time 1.58 +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 201-------------------- +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440236 41.520573 41.384872 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.148438 68.886719 67.902344 +util 93.750000 99.000000 89.000000 +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 202 : avg iteration time 49.05 seconds +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 202 : data loading time 0.00 +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 202 : avg forward pass time 13.55 +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 202 : backward pass time 10.19 +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 202 : optimizer step time 1.57 +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 202-------------------- +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440235 41.520550 41.384872 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.133789 68.886719 67.882812 +util 92.750000 97.000000 89.000000 +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 203 : avg iteration time 46.24 seconds +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 203 : data loading time 0.00 +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 203 : avg forward pass time 12.16 +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 203 : backward pass time 10.18 +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 203 : optimizer step time 1.57 +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 203-------------------- +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:21:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439720 41.518482 41.384872 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.997070 68.457031 67.843750 +util 97.000000 99.000000 91.000000 +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 204 : avg iteration time 43.72 seconds +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 204 : data loading time 0.00 +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 204 : avg forward pass time 10.88 +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 204 : backward pass time 10.18 +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 204 : optimizer step time 1.57 +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 204-------------------- +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439442 41.517258 41.384968 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.088379 108.228516 107.667969 +util 65.000000 71.000000 57.000000 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 205 : avg iteration time 161.59 seconds +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 205 : data loading time 0.00 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 205 : avg forward pass time 72.19 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 205 : backward pass time 8.39 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 205 : optimizer step time 0.40 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 205-------------------- +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6211 iter count: 1.0 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6211 iter count: 1.0 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6211 iter count: 1.0 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6211iter count: 1.0 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:24:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439443 41.517258 41.384968 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.420898 67.675781 67.335938 +util 97.250000 99.000000 94.000000 +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 206 : avg iteration time 47.37 seconds +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 206 : data loading time 0.00 +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 206 : avg forward pass time 12.73 +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 206 : backward pass time 10.21 +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 206 : optimizer step time 1.58 +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 206-------------------- +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:25:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439500 41.517498 41.384968 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.871094 67.277344 +util 93.250000 100.000000 88.000000 +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 207 : avg iteration time 46.09 seconds +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 207 : data loading time 0.00 +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 207 : avg forward pass time 12.13 +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 207 : backward pass time 10.12 +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 207 : optimizer step time 1.57 +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 207-------------------- +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:26:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439500 41.517498 41.384968 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.695312 67.277344 +util 95.000000 100.000000 89.000000 +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 208 : avg iteration time 46.19 seconds +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 208 : data loading time 0.00 +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 208 : avg forward pass time 12.18 +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 208 : backward pass time 10.13 +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 208 : optimizer step time 1.58 +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 208-------------------- +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:26:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439508 41.517502 41.384968 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.832031 67.160156 +util 94.500000 99.000000 88.000000 +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 209 : avg iteration time 48.79 seconds +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 209 : data loading time 0.00 +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 209 : avg forward pass time 13.47 +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 209 : backward pass time 10.13 +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 209 : optimizer step time 1.58 +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 209-------------------- +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:27:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439061 41.515675 41.384968 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.687500 107.603516 +util 66.000000 70.000000 62.000000 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 210 : avg iteration time 158.73 seconds +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 210 : data loading time 0.00 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 210 : avg forward pass time 70.70 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 210 : backward pass time 8.41 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 210 : optimizer step time 0.40 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 210-------------------- +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6377 iter count: 1.0 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6377 iter count: 1.0 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6377 iter count: 1.0 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6377iter count: 1.0 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:30:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439054 41.515675 41.384968 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.695312 67.335938 +util 97.000000 100.000000 90.000000 +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 211 : avg iteration time 47.25 seconds +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 211 : data loading time 0.00 +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 211 : avg forward pass time 12.70 +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 211 : backward pass time 10.13 +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 211 : optimizer step time 1.57 +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 211-------------------- +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:31:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439174 41.516155 41.384968 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499023 67.929688 67.355469 +util 92.750000 97.000000 89.000000 +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 212 : avg iteration time 48.90 seconds +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 212 : data loading time 0.00 +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 212 : avg forward pass time 13.52 +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 212 : backward pass time 10.12 +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 212 : optimizer step time 1.59 +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 212-------------------- +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:31:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439250 41.516403 41.384968 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.518555 68.007812 67.355469 +util 97.000000 99.000000 91.000000 +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 213 : avg iteration time 43.52 seconds +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 213 : data loading time 0.00 +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 213 : avg forward pass time 10.84 +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 213 : backward pass time 10.13 +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 213 : optimizer step time 1.56 +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 213-------------------- +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:32:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439236 41.516403 41.384968 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.464844 67.792969 67.355469 +util 94.250000 99.000000 86.000000 +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 214 : avg iteration time 48.95 seconds +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 214 : data loading time 0.00 +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 214 : avg forward pass time 13.48 +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 214 : backward pass time 10.21 +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 214 : optimizer step time 1.59 +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 214-------------------- +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:33:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438501 41.513435 41.384979 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.687500 107.525391 +util 62.750000 67.000000 59.000000 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 215 : avg iteration time 161.35 seconds +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 215 : data loading time 0.00 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 215 : avg forward pass time 72.03 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 215 : backward pass time 8.45 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 215 : optimizer step time 0.40 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 215-------------------- +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6992 iter count: 1.0 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6992 iter count: 1.0 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6992 iter count: 1.0 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6992iter count: 1.0 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:36:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438503 41.513416 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.500488 67.812500 67.396484 +util 96.000000 99.000000 89.000000 +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 216 : avg iteration time 47.29 seconds +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 216 : data loading time 0.00 +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 216 : avg forward pass time 12.71 +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 216 : backward pass time 10.14 +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 216 : optimizer step time 1.57 +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 216-------------------- +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438642 41.513977 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.520020 67.949219 67.376953 +util 96.000000 99.000000 92.000000 +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 217 : avg iteration time 46.34 seconds +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 217 : data loading time 0.00 +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 217 : avg forward pass time 12.20 +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 217 : backward pass time 10.12 +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 217 : optimizer step time 1.56 +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 217-------------------- +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:37:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438640 41.513992 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.524902 67.968750 67.376953 +util 90.500000 95.000000 86.000000 +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 218 : avg iteration time 48.74 seconds +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 218 : data loading time 0.00 +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 218 : avg forward pass time 13.43 +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 218 : backward pass time 10.14 +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 218 : optimizer step time 1.60 +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 218-------------------- +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:38:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438643 41.513992 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.520020 67.949219 67.376953 +util 93.250000 100.000000 88.000000 +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 219 : avg iteration time 46.18 seconds +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 219 : data loading time 0.00 +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 219 : avg forward pass time 12.16 +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 219 : backward pass time 10.15 +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 219 : optimizer step time 1.57 +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 219-------------------- +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:39:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438450 41.513199 41.384979 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.707031 107.564453 +util 68.250000 74.000000 60.000000 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 220 : avg iteration time 160.57 seconds +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 220 : data loading time 0.00 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 220 : avg forward pass time 71.64 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 220 : backward pass time 8.43 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 220 : optimizer step time 0.40 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 220-------------------- +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:42:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438456 41.513199 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.479492 67.792969 67.375000 +util 96.750000 100.000000 90.000000 +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 221 : avg iteration time 44.78 seconds +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 221 : data loading time 0.00 +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 221 : avg forward pass time 11.47 +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 221 : backward pass time 10.13 +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 221 : optimizer step time 1.56 +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 221-------------------- +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:42:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438504 41.513439 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.753906 67.335938 +util 95.250000 100.000000 89.000000 +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 222 : avg iteration time 48.64 seconds +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 222 : data loading time 0.00 +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 222 : avg forward pass time 13.38 +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 222 : backward pass time 10.14 +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 222 : optimizer step time 1.59 +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 222-------------------- +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:43:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438509 41.513439 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.469727 67.929688 67.316406 +util 95.500000 99.000000 92.000000 +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 223 : avg iteration time 48.89 seconds +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 223 : data loading time 0.00 +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 223 : avg forward pass time 13.54 +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 223 : backward pass time 10.11 +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 223 : optimizer step time 1.56 +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 223-------------------- +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:44:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438577 41.513687 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.508789 67.968750 67.355469 +util 96.250000 99.000000 89.000000 +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 224 : avg iteration time 43.47 seconds +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 224 : data loading time 0.00 +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 224 : avg forward pass time 10.80 +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 224 : backward pass time 10.15 +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 224 : optimizer step time 1.57 +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 224-------------------- +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:45:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438213 41.512257 41.384979 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.628906 107.623047 +util 63.250000 72.000000 56.000000 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 225 : avg iteration time 166.66 seconds +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 225 : data loading time 0.00 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 225 : avg forward pass time 74.73 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 225 : backward pass time 8.39 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 225 : optimizer step time 0.39 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 225-------------------- +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6504 iter count: 1.0 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6504 iter count: 1.0 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6504 iter count: 1.0 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6504iter count: 1.0 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:47:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438224 41.512280 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.479492 67.968750 67.316406 +util 94.250000 99.000000 90.000000 +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 226 : avg iteration time 50.05 seconds +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 226 : data loading time 0.00 +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 226 : avg forward pass time 14.09 +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 226 : backward pass time 10.15 +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 226 : optimizer step time 1.58 +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 226-------------------- +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:48:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438408 41.513012 41.384979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.489258 67.949219 67.335938 +util 93.750000 100.000000 88.000000 +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 227 : avg iteration time 48.75 seconds +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 227 : data loading time 0.00 +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 227 : avg forward pass time 13.44 +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 227 : backward pass time 10.13 +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 227 : optimizer step time 1.58 +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 227-------------------- +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:49:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438431 41.513012 41.385059 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.489258 67.949219 67.335938 +util 94.500000 100.000000 89.000000 +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 228 : avg iteration time 43.61 seconds +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 228 : data loading time 0.00 +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 228 : avg forward pass time 10.87 +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 228 : backward pass time 10.16 +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 228 : optimizer step time 1.59 +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 228-------------------- +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:50:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438441 41.513012 41.385109 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.503906 67.949219 67.355469 +util 96.250000 100.000000 89.000000 +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 229 : avg iteration time 48.72 seconds +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 229 : data loading time 0.00 +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 229 : avg forward pass time 13.42 +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 229 : backward pass time 10.14 +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 229 : optimizer step time 1.57 +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 229-------------------- +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:51:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438330 41.512547 41.385109 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.628906 107.544922 +util 68.750000 73.000000 66.000000 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 230 : avg iteration time 158.59 seconds +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 230 : data loading time 0.00 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 230 : avg forward pass time 70.69 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 230 : backward pass time 8.39 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 230 : optimizer step time 0.40 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 230-------------------- +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6406 iter count: 1.0 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6406 iter count: 1.0 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6406 iter count: 1.0 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6406iter count: 1.0 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:53:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438340 41.512589 41.385109 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.510254 67.851562 67.396484 +util 97.750000 100.000000 95.000000 +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 231 : avg iteration time 50.19 seconds +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 231 : data loading time 0.00 +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 231 : avg forward pass time 14.15 +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 231 : backward pass time 10.15 +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 231 : optimizer step time 1.57 +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 231-------------------- +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:54:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438468 41.513119 41.385109 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.456543 67.871094 67.318359 +util 92.000000 99.000000 86.000000 +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 232 : avg iteration time 46.07 seconds +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 232 : data loading time 0.00 +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 232 : avg forward pass time 12.11 +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 232 : backward pass time 10.15 +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 232 : optimizer step time 1.58 +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 232-------------------- +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:55:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438716 41.514111 41.385109 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.832031 67.318359 +util 94.500000 100.000000 89.000000 +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 233 : avg iteration time 43.63 seconds +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 233 : data loading time 0.00 +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 233 : avg forward pass time 10.89 +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 233 : backward pass time 10.13 +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 233 : optimizer step time 1.58 +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 233-------------------- +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:56:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438812 41.514362 41.385109 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.851562 67.318359 +util 93.000000 100.000000 87.000000 +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 234 : avg iteration time 46.18 seconds +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 234 : data loading time 0.00 +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 234 : avg forward pass time 12.17 +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 234 : backward pass time 10.13 +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 234 : optimizer step time 1.57 +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 234-------------------- +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:56:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438409 41.512627 41.385201 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.746094 107.564453 +util 66.500000 71.000000 63.000000 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 235 : avg iteration time 158.69 seconds +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 235 : data loading time 0.00 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 235 : avg forward pass time 70.75 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 235 : backward pass time 8.40 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 235 : optimizer step time 0.39 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 235-------------------- +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6211 iter count: 1.0 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6211 iter count: 1.0 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6211 iter count: 1.0 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6211iter count: 1.0 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 00:59:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438411 41.512627 41.385201 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.773438 67.355469 +util 96.250000 99.000000 93.000000 +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 236 : avg iteration time 44.79 seconds +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 236 : data loading time 0.00 +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 236 : avg forward pass time 11.47 +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 236 : backward pass time 10.14 +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 236 : optimizer step time 1.58 +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 236-------------------- +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438544 41.513123 41.385223 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.695312 67.375000 +util 94.250000 99.000000 88.000000 +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 237 : avg iteration time 43.48 seconds +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 237 : data loading time 0.00 +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 237 : avg forward pass time 10.77 +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 237 : backward pass time 10.15 +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 237 : optimizer step time 1.57 +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 237-------------------- +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:00:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438916 41.514614 41.385220 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.523438 67.968750 67.375000 +util 92.000000 97.000000 88.000000 +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 238 : avg iteration time 48.66 seconds +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 238 : data loading time 0.00 +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 238 : avg forward pass time 13.38 +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 238 : backward pass time 10.16 +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 238 : optimizer step time 1.57 +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 238-------------------- +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:01:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439098 41.515350 41.385204 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.518555 67.949219 67.375000 +util 94.000000 99.000000 87.000000 +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 239 : avg iteration time 48.94 seconds +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 239 : data loading time 0.00 +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 239 : avg forward pass time 13.50 +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 239 : backward pass time 10.15 +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 239 : optimizer step time 1.59 +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 239-------------------- +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:02:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439056 41.515182 41.385227 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.736816 107.740234 107.726562 +util 62.500000 67.000000 59.000000 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 240 : avg iteration time 161.69 seconds +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 240 : data loading time 0.00 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 240 : avg forward pass time 72.22 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 240 : backward pass time 8.41 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 240 : optimizer step time 0.40 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 240-------------------- +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6885 iter count: 1.0 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6885 iter count: 1.0 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6885 iter count: 1.0 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6885iter count: 1.0 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:05:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438938 41.514713 41.385204 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.871094 67.355469 +util 96.500000 99.000000 90.000000 +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 241 : avg iteration time 47.29 seconds +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 241 : data loading time 0.00 +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 241 : avg forward pass time 12.73 +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 241 : backward pass time 10.12 +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 241 : optimizer step time 1.56 +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 241-------------------- +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:06:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438945 41.514706 41.385223 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.474609 67.890625 67.335938 +util 94.250000 100.000000 89.000000 +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 242 : avg iteration time 43.47 seconds +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 242 : data loading time 0.00 +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 242 : avg forward pass time 10.81 +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 242 : backward pass time 10.14 +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 242 : optimizer step time 1.57 +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 242-------------------- +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:06:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438944 41.514713 41.385227 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499023 67.988281 67.335938 +util 94.750000 99.000000 89.000000 +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 243 : avg iteration time 46.17 seconds +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 243 : data loading time 0.00 +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 243 : avg forward pass time 12.17 +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 243 : backward pass time 10.13 +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 243 : optimizer step time 1.56 +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 243-------------------- +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:07:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438936 41.514706 41.385204 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499023 67.988281 67.335938 +util 91.250000 96.000000 87.000000 +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 244 : avg iteration time 43.44 seconds +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 244 : data loading time 0.00 +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 244 : avg forward pass time 10.81 +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 244 : backward pass time 10.13 +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 244 : optimizer step time 1.57 +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 244-------------------- +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:08:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438939 41.514706 41.385204 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.687500 107.505859 +util 66.500000 71.000000 63.000000 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 245 : avg iteration time 158.47 seconds +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 245 : data loading time 0.00 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 245 : avg forward pass time 70.68 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 245 : backward pass time 8.39 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 245 : optimizer step time 0.39 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 245-------------------- +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6475 iter count: 1.0 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6475 iter count: 1.0 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6475 iter count: 1.0 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6475iter count: 1.0 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:10:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438939 41.514706 41.385204 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.528320 67.988281 67.375000 +util 95.250000 100.000000 89.000000 +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 246 : avg iteration time 47.29 seconds +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 246 : data loading time 0.00 +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 246 : avg forward pass time 12.69 +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 246 : backward pass time 10.17 +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 246 : optimizer step time 1.57 +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 246-------------------- +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:11:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438975 41.514706 41.385227 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.450195 67.851562 67.316406 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 247 : avg iteration time 46.20 seconds +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 247 : data loading time 0.00 +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 247 : avg forward pass time 12.15 +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 247 : backward pass time 10.15 +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 247 : optimizer step time 1.59 +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 247-------------------- +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:12:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438978 41.514706 41.385212 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.871094 67.316406 +util 94.500000 100.000000 88.000000 +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 248 : avg iteration time 48.78 seconds +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 248 : data loading time 0.00 +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 248 : avg forward pass time 13.40 +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 248 : backward pass time 10.22 +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 248 : optimizer step time 1.59 +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 248-------------------- +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:13:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438995 41.514797 41.385204 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.871094 67.316406 +util 97.250000 99.000000 95.000000 +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 249 : avg iteration time 46.14 seconds +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 249 : data loading time 0.00 +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 249 : avg forward pass time 12.12 +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 249 : backward pass time 10.16 +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 249 : optimizer step time 1.59 +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 249-------------------- +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:14:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439006 41.514820 41.385204 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.726562 107.603516 +util 62.000000 67.000000 59.000000 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 250 : avg iteration time 166.34 seconds +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 250 : data loading time 0.00 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 250 : avg forward pass time 74.60 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 250 : backward pass time 8.37 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 250 : optimizer step time 0.40 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 250-------------------- +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6406 iter count: 1.0 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6406 iter count: 1.0 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6406 iter count: 1.0 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6406iter count: 1.0 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:16:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438892 41.514336 41.385254 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.988281 67.316406 +util 91.250000 96.000000 87.000000 +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 251 : avg iteration time 49.98 seconds +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 251 : data loading time 0.00 +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 251 : avg forward pass time 14.07 +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 251 : backward pass time 10.13 +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 251 : optimizer step time 1.56 +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 251-------------------- +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:17:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438889 41.514332 41.385231 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.851562 67.199219 +util 94.250000 99.000000 88.000000 +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 252 : avg iteration time 48.94 seconds +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 252 : data loading time 0.00 +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 252 : avg forward pass time 13.51 +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 252 : backward pass time 10.14 +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 252 : optimizer step time 1.58 +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 252-------------------- +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:18:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438952 41.514584 41.385231 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.464844 67.968750 67.296875 +util 96.250000 99.000000 89.000000 +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 253 : avg iteration time 46.18 seconds +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 253 : data loading time 0.00 +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 253 : avg forward pass time 12.17 +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 253 : backward pass time 10.12 +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 253 : optimizer step time 1.56 +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 253-------------------- +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439074 41.515083 41.385231 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.871094 67.296875 +util 96.000000 99.000000 93.000000 +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 254 : avg iteration time 46.14 seconds +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 254 : data loading time 0.00 +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 254 : avg forward pass time 12.16 +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 254 : backward pass time 10.12 +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 254 : optimizer step time 1.56 +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 254-------------------- +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438978 41.514656 41.385254 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.648438 107.525391 +util 65.000000 70.000000 58.000000 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 255 : avg iteration time 161.11 seconds +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 255 : data loading time 0.00 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 255 : avg forward pass time 71.95 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 255 : backward pass time 8.42 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 255 : optimizer step time 0.40 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 255-------------------- +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6875 iter count: 1.0 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6875 iter count: 1.0 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6875 iter count: 1.0 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6875iter count: 1.0 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:22:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438981 41.514706 41.385239 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.503906 67.773438 67.414062 +util 95.250000 99.000000 92.000000 +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 256 : avg iteration time 47.68 seconds +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 256 : data loading time 0.00 +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 256 : avg forward pass time 12.88 +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 256 : backward pass time 10.16 +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 256 : optimizer step time 1.56 +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 256-------------------- +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:23:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438971 41.514664 41.385239 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.949219 67.179688 +util 96.750000 99.000000 90.000000 +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 257 : avg iteration time 43.68 seconds +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 257 : data loading time 0.00 +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 257 : avg forward pass time 10.88 +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 257 : backward pass time 10.20 +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 257 : optimizer step time 1.57 +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 257-------------------- +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:24:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438978 41.514668 41.385239 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.391602 67.968750 67.199219 +util 94.000000 99.000000 90.000000 +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 258 : avg iteration time 48.70 seconds +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 258 : data loading time 0.00 +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 258 : avg forward pass time 13.41 +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 258 : backward pass time 10.15 +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 258 : optimizer step time 1.57 +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 258-------------------- +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:25:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439113 41.515179 41.385239 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.376953 67.968750 67.179688 +util 94.500000 100.000000 88.000000 +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 259 : avg iteration time 46.12 seconds +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 259 : data loading time 0.00 +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 259 : avg forward pass time 12.13 +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 259 : backward pass time 10.14 +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 259 : optimizer step time 1.56 +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 259-------------------- +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:25:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438869 41.514061 41.385288 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.824219 107.486328 +util 67.250000 72.000000 64.000000 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 260 : avg iteration time 158.77 seconds +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 260 : data loading time 0.00 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 260 : avg forward pass time 70.80 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 260 : backward pass time 8.42 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 260 : optimizer step time 0.40 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 260-------------------- +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6348 iter count: 1.0 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6348 iter count: 1.0 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6348 iter count: 1.0 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6348iter count: 1.0 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:28:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438880 41.514023 41.385315 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.445312 67.714844 67.355469 +util 95.500000 100.000000 89.000000 +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 261 : avg iteration time 44.78 seconds +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 261 : data loading time 0.00 +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 261 : avg forward pass time 11.42 +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 261 : backward pass time 10.13 +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 261 : optimizer step time 1.57 +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 261-------------------- +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:29:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438878 41.514023 41.385315 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.469727 67.871094 67.335938 +util 96.500000 99.000000 90.000000 +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 262 : avg iteration time 48.83 seconds +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 262 : data loading time 0.00 +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 262 : avg forward pass time 13.44 +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 262 : backward pass time 10.22 +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 262 : optimizer step time 1.57 +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 262-------------------- +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:30:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438884 41.514046 41.385315 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.929688 67.335938 +util 91.750000 97.000000 88.000000 +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 263 : avg iteration time 43.69 seconds +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 263 : data loading time 0.00 +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 263 : avg forward pass time 10.91 +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 263 : backward pass time 10.14 +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 263 : optimizer step time 1.58 +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 263-------------------- +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:30:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438905 41.514118 41.385326 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 67.968750 67.335938 +util 95.750000 100.000000 88.000000 +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 264 : avg iteration time 46.29 seconds +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 264 : data loading time 0.00 +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 264 : avg forward pass time 12.21 +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 264 : backward pass time 10.15 +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 264 : optimizer step time 1.58 +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 264-------------------- +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:31:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438871 41.513996 41.385315 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.658691 107.707031 107.642578 +util 61.000000 66.000000 57.000000 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 265 : avg iteration time 158.50 seconds +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 265 : data loading time 0.00 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 265 : avg forward pass time 70.68 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 265 : backward pass time 8.39 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 265 : optimizer step time 0.39 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 265-------------------- +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5854 iter count: 1.0 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5854 iter count: 1.0 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5854 iter count: 1.0 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5854iter count: 1.0 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:34:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438869 41.513985 41.385315 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.645508 67.988281 67.531250 +util 96.250000 99.000000 89.000000 +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 266 : avg iteration time 47.71 seconds +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 266 : data loading time 0.00 +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 266 : avg forward pass time 12.87 +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 266 : backward pass time 10.25 +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 266 : optimizer step time 1.57 +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 266-------------------- +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:34:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438886 41.514008 41.385315 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 67.988281 67.667969 +util 93.500000 98.000000 90.000000 +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 267 : avg iteration time 46.32 seconds +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 267 : data loading time 0.00 +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 267 : avg forward pass time 12.21 +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 267 : backward pass time 10.16 +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 267 : optimizer step time 1.58 +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 267-------------------- +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:35:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438872 41.513985 41.385323 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.806641 67.871094 67.785156 +util 94.500000 100.000000 89.000000 +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 268 : avg iteration time 48.88 seconds +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 268 : data loading time 0.00 +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 268 : avg forward pass time 13.49 +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 268 : backward pass time 10.16 +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 268 : optimizer step time 1.57 +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 268-------------------- +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:36:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438883 41.513985 41.385315 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.811523 67.843750 67.714844 +util 94.000000 98.000000 91.000000 +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 269 : avg iteration time 43.71 seconds +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 269 : data loading time 0.00 +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 269 : avg forward pass time 10.91 +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 269 : backward pass time 10.14 +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 269 : optimizer step time 1.59 +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 269-------------------- +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438645 41.513031 41.385315 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.628906 107.603516 +util 62.250000 67.000000 59.000000 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 270 : avg iteration time 158.58 seconds +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 270 : data loading time 0.00 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 270 : avg forward pass time 70.72 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 270 : backward pass time 8.38 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 270 : optimizer step time 0.40 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 270-------------------- +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6387 iter count: 1.0 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6387 iter count: 1.0 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6387 iter count: 1.0 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6387iter count: 1.0 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:39:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438641 41.513008 41.385315 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.714844 67.357422 +util 95.750000 100.000000 88.000000 +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 271 : avg iteration time 49.88 seconds +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 271 : data loading time 0.00 +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 271 : avg forward pass time 13.99 +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 271 : backward pass time 10.15 +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 271 : optimizer step time 1.58 +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 271-------------------- +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:40:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438902 41.514004 41.385342 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.441895 67.812500 67.318359 +util 94.500000 100.000000 88.000000 +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 272 : avg iteration time 43.44 seconds +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 272 : data loading time 0.00 +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 272 : avg forward pass time 10.78 +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 272 : backward pass time 10.16 +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 272 : optimizer step time 1.57 +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 272-------------------- +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:41:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438921 41.514004 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.524902 68.027344 67.357422 +util 98.000000 100.000000 96.000000 +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 273 : avg iteration time 48.54 seconds +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 273 : data loading time 0.00 +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 273 : avg forward pass time 13.34 +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 273 : backward pass time 10.14 +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 273 : optimizer step time 1.57 +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 273-------------------- +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:42:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438927 41.514004 41.385456 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.968750 67.318359 +util 96.250000 99.000000 89.000000 +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 274 : avg iteration time 43.43 seconds +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 274 : data loading time 0.00 +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 274 : avg forward pass time 10.78 +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 274 : backward pass time 10.16 +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 274 : optimizer step time 1.57 +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 274-------------------- +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:43:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438697 41.513073 41.385441 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.687500 107.642578 +util 61.750000 67.000000 59.000000 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 275 : avg iteration time 160.90 seconds +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 275 : data loading time 0.00 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 275 : avg forward pass time 71.85 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 275 : backward pass time 8.40 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 275 : optimizer step time 0.40 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 275-------------------- +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6523 iter count: 1.0 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6523 iter count: 1.0 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6523 iter count: 1.0 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6523iter count: 1.0 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:45:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438699 41.513065 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.524902 67.968750 67.376953 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 276 : avg iteration time 49.93 seconds +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 276 : data loading time 0.00 +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 276 : avg forward pass time 14.02 +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 276 : backward pass time 10.16 +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 276 : optimizer step time 1.56 +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 276-------------------- +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:46:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438878 41.513813 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.539551 68.027344 67.376953 +util 93.250000 98.000000 90.000000 +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 277 : avg iteration time 48.66 seconds +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 277 : data loading time 0.00 +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 277 : avg forward pass time 13.39 +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 277 : backward pass time 10.15 +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 277 : optimizer step time 1.57 +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 277-------------------- +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:47:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438937 41.514042 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.539551 67.968750 67.396484 +util 95.500000 99.000000 88.000000 +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 278 : avg iteration time 48.68 seconds +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 278 : data loading time 0.00 +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 278 : avg forward pass time 13.39 +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 278 : backward pass time 10.14 +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 278 : optimizer step time 1.58 +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 278-------------------- +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:48:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:48:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438941 41.514042 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.534668 68.007812 67.376953 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 01:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 279 : avg iteration time 43.46 seconds +[May 4, 2026 - 01:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 279 : data loading time 0.00 +[May 4, 2026 - 01:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 279 : avg forward pass time 10.80 +[May 4, 2026 - 01:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 279 : backward pass time 10.13 +[May 4, 2026 - 01:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 279 : optimizer step time 1.58 +[May 4, 2026 - 01:48:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 279-------------------- +[May 4, 2026 - 01:48:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:48:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:48:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:48:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438761 41.513298 41.385441 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.839355 108.488281 107.623047 +util 62.500000 66.000000 57.000000 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 280 : avg iteration time 163.25 seconds +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 280 : data loading time 0.00 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 280 : avg forward pass time 73.04 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 280 : backward pass time 8.39 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 280 : optimizer step time 0.40 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 280-------------------- +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6416 iter count: 1.0 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6416 iter count: 1.0 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6416 iter count: 1.0 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6416iter count: 1.0 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:51:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438764 41.513298 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.495605 67.968750 67.337891 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 281 : avg iteration time 44.55 seconds +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 281 : data loading time 0.00 +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 281 : avg forward pass time 11.35 +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 281 : backward pass time 10.14 +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 281 : optimizer step time 1.58 +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 281-------------------- +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:52:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438848 41.513641 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.495605 67.968750 67.337891 +util 96.750000 99.000000 90.000000 +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 282 : avg iteration time 48.77 seconds +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 282 : data loading time 0.00 +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 282 : avg forward pass time 13.45 +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 282 : backward pass time 10.14 +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 282 : optimizer step time 1.58 +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 282-------------------- +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:53:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439027 41.514385 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.968750 67.318359 +util 95.750000 100.000000 90.000000 +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 283 : avg iteration time 48.63 seconds +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 283 : data loading time 0.00 +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 283 : avg forward pass time 13.39 +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 283 : backward pass time 10.12 +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 283 : optimizer step time 1.58 +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 283-------------------- +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439027 41.514385 41.385441 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.968750 67.318359 +util 96.000000 99.000000 93.000000 +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 284 : avg iteration time 46.09 seconds +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 284 : data loading time 0.00 +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 284 : avg forward pass time 12.12 +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 284 : backward pass time 10.15 +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 284 : optimizer step time 1.57 +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 284-------------------- +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438710 41.513081 41.385456 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.702637 107.707031 107.701172 +util 66.250000 71.000000 63.000000 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 285 : avg iteration time 160.63 seconds +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 285 : data loading time 0.00 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 285 : avg forward pass time 71.72 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 285 : backward pass time 8.41 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 285 : optimizer step time 0.41 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 285-------------------- +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6777 iter count: 1.0 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6777 iter count: 1.0 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6777 iter count: 1.0 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6777iter count: 1.0 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:57:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438725 41.513081 41.385479 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.510254 67.968750 67.357422 +util 92.250000 97.000000 89.000000 +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 286 : avg iteration time 44.72 seconds +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 286 : data loading time 0.00 +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 286 : avg forward pass time 11.33 +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 286 : backward pass time 10.13 +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 286 : optimizer step time 1.57 +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 286-------------------- +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:58:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438868 41.513699 41.385483 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.554199 68.027344 67.396484 +util 95.250000 100.000000 89.000000 +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 287 : avg iteration time 48.54 seconds +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 287 : data loading time 0.00 +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 287 : avg forward pass time 13.34 +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 287 : backward pass time 10.14 +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 287 : optimizer step time 1.57 +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 287-------------------- +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:58:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439183 41.514961 41.385483 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.534668 67.949219 67.396484 +util 95.000000 100.000000 89.000000 +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 288 : avg iteration time 48.60 seconds +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 288 : data loading time 0.00 +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 288 : avg forward pass time 13.38 +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 288 : backward pass time 10.13 +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 288 : optimizer step time 1.57 +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 288-------------------- +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 01:59:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439252 41.515213 41.385483 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.554199 68.027344 67.396484 +util 95.250000 100.000000 89.000000 +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 289 : avg iteration time 48.60 seconds +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 289 : data loading time 0.00 +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 289 : avg forward pass time 13.39 +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 289 : backward pass time 10.14 +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 289 : optimizer step time 1.56 +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 289-------------------- +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:00:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439079 41.514507 41.385483 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.687988 107.707031 107.681641 +util 63.750000 73.000000 56.000000 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 290 : avg iteration time 163.04 seconds +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 290 : data loading time 0.00 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 290 : avg forward pass time 72.91 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 290 : backward pass time 8.44 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 290 : optimizer step time 0.40 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 290-------------------- +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6006 iter count: 1.0 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6006 iter count: 1.0 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6006 iter count: 1.0 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6006iter count: 1.0 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:03:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438941 41.513947 41.385483 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499023 67.988281 67.335938 +util 94.500000 100.000000 88.000000 +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 291 : avg iteration time 44.55 seconds +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 291 : data loading time 0.00 +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 291 : avg forward pass time 11.33 +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 291 : backward pass time 10.15 +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 291 : optimizer step time 1.57 +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 291-------------------- +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:04:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438949 41.513947 41.385544 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.832031 67.335938 +util 94.750000 100.000000 88.000000 +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 292 : avg iteration time 48.59 seconds +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 292 : data loading time 0.00 +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 292 : avg forward pass time 13.36 +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 292 : backward pass time 10.15 +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 292 : optimizer step time 1.58 +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 292-------------------- +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:04:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438951 41.513947 41.385532 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.479492 67.968750 67.316406 +util 96.500000 99.000000 94.000000 +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 293 : avg iteration time 46.13 seconds +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 293 : data loading time 0.00 +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 293 : avg forward pass time 12.14 +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 293 : backward pass time 10.13 +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 293 : optimizer step time 1.58 +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 293-------------------- +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:05:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.438960 41.513947 41.385555 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.479492 67.851562 67.355469 +util 94.500000 100.000000 88.000000 +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 294 : avg iteration time 46.07 seconds +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 294 : data loading time 0.00 +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 294 : avg forward pass time 12.10 +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 294 : backward pass time 10.15 +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 294 : optimizer step time 1.58 +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 294-------------------- +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:06:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439148 41.514721 41.385540 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.589844 107.564453 +util 64.500000 70.000000 58.000000 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 295 : avg iteration time 163.41 seconds +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 295 : data loading time 0.00 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 295 : avg forward pass time 73.11 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 295 : backward pass time 8.40 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 295 : optimizer step time 0.40 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 295-------------------- +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6221 iter count: 1.0 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6221 iter count: 1.0 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6221 iter count: 1.0 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6221iter count: 1.0 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:09:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439008 41.514156 41.385559 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.495605 67.792969 67.396484 +util 95.000000 99.000000 87.000000 +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 296 : avg iteration time 44.54 seconds +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 296 : data loading time 0.00 +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 296 : avg forward pass time 11.33 +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 296 : backward pass time 10.15 +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 296 : optimizer step time 1.58 +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 296-------------------- +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:09:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439031 41.514248 41.385559 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.568848 67.792969 67.494141 +util 92.500000 99.000000 86.000000 +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 297 : avg iteration time 43.48 seconds +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 297 : data loading time 0.00 +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 297 : avg forward pass time 10.80 +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 297 : backward pass time 10.13 +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 297 : optimizer step time 1.59 +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 297-------------------- +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:10:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439218 41.514988 41.385559 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.792969 67.337891 +util 94.250000 99.000000 91.000000 +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 298 : avg iteration time 48.65 seconds +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 298 : data loading time 0.00 +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 298 : avg forward pass time 13.39 +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 298 : backward pass time 10.14 +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 298 : optimizer step time 1.58 +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 298-------------------- +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:11:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439221 41.515003 41.385559 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.427246 67.988281 67.240234 +util 93.250000 100.000000 88.000000 +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 299 : avg iteration time 45.94 seconds +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 299 : data loading time 0.00 +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 299 : avg forward pass time 12.04 +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 299 : backward pass time 10.16 +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 299 : optimizer step time 1.56 +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 299-------------------- +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:12:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439218 41.514988 41.385559 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.687500 107.583984 +util 65.500000 70.000000 62.000000 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 300 : avg iteration time 160.68 seconds +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 300 : data loading time 0.00 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 300 : avg forward pass time 71.76 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 300 : backward pass time 8.42 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 300 : optimizer step time 0.39 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 300-------------------- +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6328 iter count: 1.0 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6328 iter count: 1.0 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6328 iter count: 1.0 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6328iter count: 1.0 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 02:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 02:15:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.27 seconds +[May 4, 2026 - 02:15:03 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 300 +[May 4, 2026 - 02:15:03 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000300 +[May 4, 2026 - 02:15:45 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 300 +[May 4, 2026 - 02:15:45 | INFO | fastgen.trainer:validate:358 ] Validation iteration 300 +[May 4, 2026 - 02:15:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:15:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 4, 2026 - 02:15:51 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.214 +[May 4, 2026 - 02:15:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:15:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 02:15:56 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.642 +[May 4, 2026 - 02:15:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:15:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 02:16:01 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.501 +[May 4, 2026 - 02:16:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:16:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 02:16:06 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.265 +[May 4, 2026 - 02:16:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:16:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 02:16:11 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.654 +[May 4, 2026 - 02:16:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:16:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 02:16:16 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.850 +[May 4, 2026 - 02:16:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:16:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 02:16:21 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=7.013 +[May 4, 2026 - 02:16:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:16:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 02:16:26 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.737 +[May 4, 2026 - 02:16:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:16:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 02:16:31 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.756 +[May 4, 2026 - 02:16:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 02:16:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 02:16:36 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.656 +[May 4, 2026 - 02:16:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 300-------------------- +[May 4, 2026 - 02:16:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 02:16:52 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 300 +[May 4, 2026 - 02:16:52 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.429 (n=10) +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440979 41.520851 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.324219 68.652344 68.214844 +util 93.500000 100.000000 87.000000 +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 301 : avg iteration time 168.33 seconds +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 301 : data loading time 0.00 +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 301 : avg forward pass time 11.68 +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 301 : backward pass time 10.13 +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 301 : optimizer step time 1.58 +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 301-------------------- +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:17:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440973 41.520840 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.377930 68.691406 68.273438 +util 96.000000 100.000000 91.000000 +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 302 : avg iteration time 46.17 seconds +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 302 : data loading time 0.00 +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 302 : avg forward pass time 12.11 +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 302 : backward pass time 10.19 +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 302 : optimizer step time 1.57 +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 302-------------------- +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:18:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440708 41.519779 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.373047 68.671875 68.273438 +util 95.500000 99.000000 89.000000 +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 303 : avg iteration time 46.33 seconds +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 303 : data loading time 0.00 +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 303 : avg forward pass time 12.17 +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 303 : backward pass time 10.20 +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 303 : optimizer step time 1.58 +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 303-------------------- +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:19:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440715 41.519783 41.385975 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.338867 68.652344 68.234375 +util 93.250000 99.000000 87.000000 +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 304 : avg iteration time 48.88 seconds +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 304 : data loading time 0.00 +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 304 : avg forward pass time 13.49 +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 304 : backward pass time 10.15 +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 304 : optimizer step time 1.57 +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 304-------------------- +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:19:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439746 41.515930 41.385952 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.687500 107.525391 +util 62.500000 72.000000 54.000000 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 305 : avg iteration time 165.79 seconds +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 305 : data loading time 0.00 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 305 : avg forward pass time 74.30 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 305 : backward pass time 8.39 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 305 : optimizer step time 0.40 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 305-------------------- +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6611 iter count: 1.0 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6611 iter count: 1.0 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6611 iter count: 1.0 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6611iter count: 1.0 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:22:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439761 41.515984 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.611328 67.628906 67.558594 +util 96.750000 99.000000 90.000000 +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 306 : avg iteration time 44.74 seconds +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 306 : data loading time 0.00 +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 306 : avg forward pass time 11.45 +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 306 : backward pass time 10.13 +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 306 : optimizer step time 1.57 +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 306-------------------- +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:23:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439884 41.516479 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.621094 67.667969 67.480469 +util 96.250000 100.000000 91.000000 +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 307 : avg iteration time 46.05 seconds +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 307 : data loading time 0.00 +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 307 : avg forward pass time 12.10 +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 307 : backward pass time 10.14 +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 307 : optimizer step time 1.58 +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 307-------------------- +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:24:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440070 41.517227 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.704102 67.746094 67.578125 +util 95.000000 99.000000 92.000000 +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 308 : avg iteration time 46.07 seconds +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 308 : data loading time 0.00 +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 308 : avg forward pass time 12.09 +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 308 : backward pass time 10.16 +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 308 : optimizer step time 1.59 +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 308-------------------- +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:25:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440072 41.517227 41.385960 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.616211 67.648438 67.519531 +util 91.250000 96.000000 88.000000 +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 309 : avg iteration time 45.95 seconds +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 309 : data loading time 0.00 +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 309 : avg forward pass time 12.03 +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 309 : backward pass time 10.13 +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 309 : optimizer step time 1.59 +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 309-------------------- +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:25:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439558 41.515179 41.385952 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.678223 107.726562 107.662109 +util 67.250000 72.000000 63.000000 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 310 : avg iteration time 157.89 seconds +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 310 : data loading time 0.00 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 310 : avg forward pass time 70.35 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 310 : backward pass time 8.43 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 310 : optimizer step time 0.40 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 310-------------------- +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5986 iter count: 1.0 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5986 iter count: 1.0 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5986 iter count: 1.0 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5986iter count: 1.0 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:28:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439565 41.515194 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.656250 67.318359 +util 95.750000 100.000000 90.000000 +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 311 : avg iteration time 47.06 seconds +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 311 : data loading time 0.00 +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 311 : avg forward pass time 12.60 +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 311 : backward pass time 10.15 +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 311 : optimizer step time 1.56 +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 311-------------------- +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439632 41.515427 41.385975 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.368652 67.578125 67.298828 +util 97.000000 99.000000 95.000000 +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 312 : avg iteration time 45.92 seconds +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 312 : data loading time 0.00 +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 312 : avg forward pass time 12.03 +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 312 : backward pass time 10.16 +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 312 : optimizer step time 1.56 +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 312-------------------- +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:29:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439940 41.516682 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.500000 67.435547 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 313 : avg iteration time 46.08 seconds +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 313 : data loading time 0.00 +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 313 : avg forward pass time 12.10 +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 313 : backward pass time 10.17 +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 313 : optimizer step time 1.57 +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 313-------------------- +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:30:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439949 41.516697 41.385952 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412598 67.578125 67.357422 +util 97.250000 100.000000 95.000000 +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 314 : avg iteration time 43.32 seconds +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 314 : data loading time 0.00 +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 314 : avg forward pass time 10.74 +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 314 : backward pass time 10.13 +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 314 : optimizer step time 1.57 +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 314-------------------- +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:31:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439255 41.513927 41.385952 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.687500 107.642578 +util 67.000000 72.000000 63.000000 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 315 : avg iteration time 160.40 seconds +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 315 : data loading time 0.00 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 315 : avg forward pass time 71.60 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 315 : backward pass time 8.38 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 315 : optimizer step time 0.40 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 315-------------------- +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6309 iter count: 1.0 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6309 iter count: 1.0 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6309 iter count: 1.0 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6309iter count: 1.0 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:34:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439264 41.513939 41.385956 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396484 67.636719 67.316406 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 316 : avg iteration time 47.09 seconds +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 316 : data loading time 0.00 +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 316 : avg forward pass time 12.62 +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 316 : backward pass time 10.13 +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 316 : optimizer step time 1.57 +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 316-------------------- +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:34:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439392 41.514427 41.385979 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.357422 67.480469 67.316406 +util 91.500000 96.000000 88.000000 +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 317 : avg iteration time 43.43 seconds +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 317 : data loading time 0.00 +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 317 : avg forward pass time 10.78 +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 317 : backward pass time 10.14 +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 317 : optimizer step time 1.57 +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 317-------------------- +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439761 41.515926 41.385956 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396484 67.519531 67.355469 +util 96.250000 100.000000 89.000000 +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 318 : avg iteration time 46.12 seconds +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 318 : data loading time 0.00 +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 318 : avg forward pass time 12.13 +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 318 : backward pass time 10.13 +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 318 : optimizer step time 1.58 +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 318-------------------- +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:36:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439784 41.515926 41.386021 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.357422 67.480469 67.316406 +util 96.750000 99.000000 90.000000 +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 319 : avg iteration time 43.40 seconds +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 319 : data loading time 0.00 +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 319 : avg forward pass time 10.77 +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 319 : backward pass time 10.14 +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 319 : optimizer step time 1.58 +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 319-------------------- +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:37:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439390 41.514339 41.385998 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.628906 107.564453 +util 67.750000 74.000000 60.000000 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 320 : avg iteration time 163.22 seconds +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 320 : data loading time 0.00 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 320 : avg forward pass time 73.03 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 320 : backward pass time 8.42 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 320 : optimizer step time 0.39 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 320-------------------- +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6680 iter count: 1.0 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6680 iter count: 1.0 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6680 iter count: 1.0 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6680iter count: 1.0 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:39:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439389 41.514355 41.385998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.378418 67.441406 67.357422 +util 97.750000 99.000000 96.000000 +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 321 : avg iteration time 44.56 seconds +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 321 : data loading time 0.00 +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 321 : avg forward pass time 11.35 +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 321 : backward pass time 10.14 +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 321 : optimizer step time 1.58 +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 321-------------------- +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:40:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439547 41.514961 41.386021 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358887 67.421875 67.337891 +util 97.000000 99.000000 95.000000 +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 322 : avg iteration time 48.59 seconds +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 322 : data loading time 0.00 +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 322 : avg forward pass time 13.35 +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 322 : backward pass time 10.16 +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 322 : optimizer step time 1.57 +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 322-------------------- +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439607 41.515198 41.386021 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.349121 67.441406 67.318359 +util 97.000000 99.000000 95.000000 +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 323 : avg iteration time 48.80 seconds +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 323 : data loading time 0.00 +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 323 : avg forward pass time 13.45 +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 323 : backward pass time 10.15 +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 323 : optimizer step time 1.58 +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 323-------------------- +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:42:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439608 41.515198 41.386021 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.378418 67.441406 67.357422 +util 96.250000 99.000000 89.000000 +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 324 : avg iteration time 43.45 seconds +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 324 : data loading time 0.00 +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 324 : avg forward pass time 10.79 +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 324 : backward pass time 10.16 +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 324 : optimizer step time 1.57 +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 324-------------------- +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:42:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439171 41.513462 41.385998 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.746094 107.544922 +util 65.750000 70.000000 62.000000 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 325 : avg iteration time 158.15 seconds +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 325 : data loading time 0.00 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 325 : avg forward pass time 70.48 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 325 : backward pass time 8.40 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 325 : optimizer step time 0.40 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 325-------------------- +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5732 iter count: 1.0 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5732 iter count: 1.0 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5732 iter count: 1.0 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5732iter count: 1.0 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:45:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439166 41.513462 41.385998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.694336 67.804688 67.363281 +util 98.500000 100.000000 97.000000 +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 326 : avg iteration time 47.04 seconds +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 326 : data loading time 0.00 +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 326 : avg forward pass time 12.59 +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 326 : backward pass time 10.15 +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 326 : optimizer step time 1.57 +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 326-------------------- +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:46:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439351 41.514202 41.385998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777344 67.843750 67.578125 +util 95.000000 100.000000 89.000000 +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 327 : avg iteration time 48.65 seconds +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 327 : data loading time 0.00 +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 327 : avg forward pass time 13.38 +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 327 : backward pass time 10.16 +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 327 : optimizer step time 1.56 +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 327-------------------- +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:47:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439488 41.514729 41.385998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.679688 67.707031 67.597656 +util 96.750000 99.000000 94.000000 +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 328 : avg iteration time 48.72 seconds +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 328 : data loading time 0.00 +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 328 : avg forward pass time 13.43 +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 328 : backward pass time 10.15 +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 328 : optimizer step time 1.57 +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 328-------------------- +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:47:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439539 41.514954 41.385998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.791992 67.882812 67.519531 +util 95.500000 99.000000 92.000000 +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 329 : avg iteration time 45.98 seconds +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 329 : data loading time 0.00 +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 329 : avg forward pass time 12.06 +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 329 : backward pass time 10.16 +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 329 : optimizer step time 1.56 +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 329-------------------- +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:48:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439069 41.513054 41.385998 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.648438 107.564453 +util 67.250000 72.000000 63.000000 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 330 : avg iteration time 166.04 seconds +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 330 : data loading time 0.00 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 330 : avg forward pass time 74.44 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 330 : backward pass time 8.40 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 330 : optimizer step time 0.40 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 330-------------------- +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6602 iter count: 1.0 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6602 iter count: 1.0 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6602 iter count: 1.0 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6602iter count: 1.0 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:51:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439126 41.513302 41.385998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.500000 67.376953 +util 94.500000 99.000000 91.000000 +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 331 : avg iteration time 46.95 seconds +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 331 : data loading time 0.00 +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 331 : avg forward pass time 12.54 +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 331 : backward pass time 10.14 +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 331 : optimizer step time 1.57 +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 331-------------------- +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:52:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439333 41.514080 41.386024 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.744629 67.826172 67.500000 +util 95.500000 99.000000 88.000000 +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 332 : avg iteration time 48.66 seconds +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 332 : data loading time 0.00 +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 332 : avg forward pass time 13.40 +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 332 : backward pass time 10.16 +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 332 : optimizer step time 1.57 +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 332-------------------- +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:53:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439563 41.515038 41.386002 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.437012 67.500000 67.416016 +util 95.250000 99.000000 92.000000 +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 333 : avg iteration time 43.35 seconds +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 333 : data loading time 0.00 +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 333 : avg forward pass time 10.74 +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 333 : backward pass time 10.16 +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 333 : optimizer step time 1.58 +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 333-------------------- +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:53:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439631 41.515308 41.386002 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.422363 67.500000 67.396484 +util 90.750000 95.000000 87.000000 +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 334 : avg iteration time 48.74 seconds +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 334 : data loading time 0.00 +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 334 : avg forward pass time 13.44 +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 334 : backward pass time 10.13 +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 334 : optimizer step time 1.58 +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 334-------------------- +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:54:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439155 41.513386 41.386009 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.628906 107.544922 +util 65.000000 70.000000 61.000000 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 335 : avg iteration time 163.27 seconds +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 335 : data loading time 0.00 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 335 : avg forward pass time 73.03 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 335 : backward pass time 8.42 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 335 : optimizer step time 0.40 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 335-------------------- +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6680 iter count: 1.0 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6680 iter count: 1.0 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6680 iter count: 1.0 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6680iter count: 1.0 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:57:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439183 41.513386 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.708984 67.656250 +util 93.750000 98.000000 90.000000 +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 336 : avg iteration time 47.16 seconds +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 336 : data loading time 0.00 +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 336 : avg forward pass time 12.66 +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 336 : backward pass time 10.15 +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 336 : optimizer step time 1.56 +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 336-------------------- +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:58:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439304 41.513882 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.787109 67.421875 +util 93.250000 98.000000 89.000000 +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 337 : avg iteration time 43.39 seconds +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 337 : data loading time 0.00 +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 337 : avg forward pass time 10.78 +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 337 : backward pass time 10.14 +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 337 : optimizer step time 1.58 +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 337-------------------- +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:58:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439486 41.514614 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.690918 67.728516 67.578125 +util 91.250000 96.000000 87.000000 +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 338 : avg iteration time 48.58 seconds +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 338 : data loading time 0.00 +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 338 : avg forward pass time 13.35 +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 338 : backward pass time 10.14 +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 338 : optimizer step time 1.58 +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 338-------------------- +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 02:59:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439548 41.514866 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.720215 67.787109 67.519531 +util 90.500000 95.000000 87.000000 +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 339 : avg iteration time 46.00 seconds +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 339 : data loading time 0.00 +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 339 : avg forward pass time 12.07 +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 339 : backward pass time 10.13 +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 339 : optimizer step time 1.58 +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 339-------------------- +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:00:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439135 41.513210 41.386009 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.946777 108.052734 107.628906 +util 69.250000 74.000000 65.000000 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 340 : avg iteration time 157.72 seconds +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 340 : data loading time 0.00 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 340 : avg forward pass time 70.28 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 340 : backward pass time 8.41 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 340 : optimizer step time 0.40 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 340-------------------- +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6436 iter count: 1.0 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6436 iter count: 1.0 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6436 iter count: 1.0 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6436iter count: 1.0 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:03:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439135 41.513210 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.681152 67.728516 67.539062 +util 96.000000 100.000000 91.000000 +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 341 : avg iteration time 44.47 seconds +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 341 : data loading time 0.00 +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 341 : avg forward pass time 11.31 +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 341 : backward pass time 10.16 +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 341 : optimizer step time 1.57 +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 341-------------------- +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439321 41.513943 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.661621 67.689453 67.578125 +util 92.750000 97.000000 89.000000 +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 342 : avg iteration time 43.44 seconds +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 342 : data loading time 0.00 +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 342 : avg forward pass time 10.78 +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 342 : backward pass time 10.14 +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 342 : optimizer step time 1.58 +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 342-------------------- +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439571 41.514938 41.386017 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.676270 67.689453 67.636719 +util 96.000000 99.000000 93.000000 +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 343 : avg iteration time 45.84 seconds +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 343 : data loading time 0.00 +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 343 : avg forward pass time 12.01 +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 343 : backward pass time 10.14 +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 343 : optimizer step time 1.56 +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 343-------------------- +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:05:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439588 41.514946 41.386028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.681152 67.689453 67.656250 +util 90.250000 95.000000 86.000000 +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 344 : avg iteration time 48.53 seconds +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 344 : data loading time 0.00 +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 344 : avg forward pass time 13.34 +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 344 : backward pass time 10.14 +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 344 : optimizer step time 1.57 +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 344-------------------- +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:06:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439589 41.514969 41.386028 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.220215 108.365234 107.785156 +util 62.500000 67.000000 58.000000 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 345 : avg iteration time 157.43 seconds +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 345 : data loading time 0.00 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 345 : avg forward pass time 70.14 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 345 : backward pass time 8.40 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 345 : optimizer step time 0.39 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 345-------------------- +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5879 iter count: 1.0 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5879 iter count: 1.0 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5879 iter count: 1.0 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5879iter count: 1.0 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:08:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439451 41.514465 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.651855 67.689453 67.539062 +util 94.750000 99.000000 91.000000 +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 346 : avg iteration time 49.61 seconds +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 346 : data loading time 0.00 +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 346 : avg forward pass time 13.88 +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 346 : backward pass time 10.13 +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 346 : optimizer step time 1.58 +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 346-------------------- +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:09:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439454 41.514465 41.386013 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.661621 67.689453 67.578125 +util 96.250000 99.000000 89.000000 +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 347 : avg iteration time 45.88 seconds +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 347 : data loading time 0.00 +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 347 : avg forward pass time 12.02 +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 347 : backward pass time 10.15 +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 347 : optimizer step time 1.57 +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 347-------------------- +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439459 41.514465 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.637207 67.708984 67.421875 +util 95.500000 99.000000 88.000000 +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 348 : avg iteration time 48.58 seconds +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 348 : data loading time 0.00 +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 348 : avg forward pass time 13.36 +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 348 : backward pass time 10.15 +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 348 : optimizer step time 1.57 +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 348-------------------- +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:11:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439461 41.514481 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.632324 67.708984 67.402344 +util 93.750000 98.000000 90.000000 +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 349 : avg iteration time 43.32 seconds +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 349 : data loading time 0.00 +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 349 : avg forward pass time 10.72 +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 349 : backward pass time 10.16 +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 349 : optimizer step time 1.57 +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 349-------------------- +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:11:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439457 41.514465 41.386021 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.648438 107.623047 +util 67.500000 72.000000 63.000000 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 350 : avg iteration time 157.47 seconds +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 350 : data loading time 0.00 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 350 : avg forward pass time 70.17 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 350 : backward pass time 8.39 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 350 : optimizer step time 0.40 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 350-------------------- +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6123 iter count: 1.0 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6123 iter count: 1.0 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6123 iter count: 1.0 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6123iter count: 1.0 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:14:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439425 41.514336 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.748047 67.539062 +util 95.500000 99.000000 88.000000 +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 351 : avg iteration time 47.01 seconds +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 351 : data loading time 0.00 +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 351 : avg forward pass time 12.56 +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 351 : backward pass time 10.15 +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 351 : optimizer step time 1.56 +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 351-------------------- +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:15:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439429 41.514336 41.386009 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.686035 67.767578 67.441406 +util 95.500000 99.000000 92.000000 +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 352 : avg iteration time 45.95 seconds +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 352 : data loading time 0.00 +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 352 : avg forward pass time 12.03 +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 352 : backward pass time 10.16 +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 352 : optimizer step time 1.58 +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 352-------------------- +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:16:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439483 41.514587 41.386013 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.681152 67.728516 67.539062 +util 96.250000 99.000000 89.000000 +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 353 : avg iteration time 48.47 seconds +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 353 : data loading time 0.00 +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 353 : avg forward pass time 13.31 +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 353 : backward pass time 10.15 +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 353 : optimizer step time 1.57 +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 353-------------------- +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439561 41.514851 41.386013 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.661621 67.728516 67.460938 +util 94.000000 99.000000 90.000000 +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 354 : avg iteration time 43.32 seconds +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 354 : data loading time 0.00 +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 354 : avg forward pass time 10.71 +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 354 : backward pass time 10.16 +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 354 : optimizer step time 1.57 +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 354-------------------- +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439614 41.515079 41.386013 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.707031 107.564453 +util 61.750000 65.000000 55.000000 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 355 : avg iteration time 160.18 seconds +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 355 : data loading time 0.00 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 355 : avg forward pass time 71.51 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 355 : backward pass time 8.40 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 355 : optimizer step time 0.40 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 355-------------------- +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5977 iter count: 1.0 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5977 iter count: 1.0 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5977 iter count: 1.0 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5977iter count: 1.0 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:20:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:21:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439612 41.515079 41.386036 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.611328 67.648438 67.500000 +util 90.500000 95.000000 87.000000 +[May 4, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 356 : avg iteration time 46.99 seconds +[May 4, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 356 : data loading time 0.00 +[May 4, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 356 : avg forward pass time 12.56 +[May 4, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 356 : backward pass time 10.13 +[May 4, 2026 - 03:21:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 356 : optimizer step time 1.55 +[May 4, 2026 - 03:21:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 356-------------------- +[May 4, 2026 - 03:21:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 03:21:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 03:21:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:21:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439619 41.515095 41.386028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.572266 67.628906 67.402344 +util 91.750000 96.000000 88.000000 +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 357 : avg iteration time 43.22 seconds +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 357 : data loading time 0.00 +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 357 : avg forward pass time 10.68 +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 357 : backward pass time 10.14 +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 357 : optimizer step time 1.57 +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 357-------------------- +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:21:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439623 41.515079 41.386028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.804688 67.597656 +util 94.000000 100.000000 87.000000 +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 358 : avg iteration time 43.32 seconds +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 358 : data loading time 0.00 +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 358 : avg forward pass time 10.73 +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 358 : backward pass time 10.14 +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 358 : optimizer step time 1.58 +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 358-------------------- +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:22:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439695 41.515327 41.386028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.684570 67.707031 67.617188 +util 92.250000 97.000000 88.000000 +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 359 : avg iteration time 43.28 seconds +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 359 : data loading time 0.00 +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 359 : avg forward pass time 10.71 +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 359 : backward pass time 10.13 +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 359 : optimizer step time 1.58 +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 359-------------------- +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:23:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439508 41.514587 41.386028 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.804688 107.525391 +util 66.500000 71.000000 62.000000 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 360 : avg iteration time 160.06 seconds +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 360 : data loading time 0.00 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 360 : avg forward pass time 71.46 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 360 : backward pass time 8.38 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 360 : optimizer step time 0.39 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 360-------------------- +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6221 iter count: 1.0 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6221 iter count: 1.0 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6221 iter count: 1.0 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6221iter count: 1.0 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:25:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439509 41.514587 41.386040 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.705566 67.728516 67.636719 +util 92.250000 97.000000 88.000000 +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 361 : avg iteration time 46.94 seconds +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 361 : data loading time 0.00 +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 361 : avg forward pass time 12.53 +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 361 : backward pass time 10.15 +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 361 : optimizer step time 1.58 +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 361-------------------- +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:26:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439507 41.514603 41.386028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.656738 67.708984 67.500000 +util 94.500000 100.000000 88.000000 +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 362 : avg iteration time 43.33 seconds +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 362 : data loading time 0.00 +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 362 : avg forward pass time 10.74 +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 362 : backward pass time 10.15 +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 362 : optimizer step time 1.57 +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 362-------------------- +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:27:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439519 41.514603 41.386032 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.671387 67.728516 67.500000 +util 97.500000 100.000000 95.000000 +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 363 : avg iteration time 45.95 seconds +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 363 : data loading time 0.00 +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 363 : avg forward pass time 12.03 +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 363 : backward pass time 10.16 +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 363 : optimizer step time 1.58 +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 363-------------------- +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:28:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439512 41.514603 41.386032 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.710449 67.728516 67.656250 +util 94.500000 99.000000 89.000000 +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 364 : avg iteration time 43.26 seconds +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 364 : data loading time 0.00 +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 364 : avg forward pass time 10.70 +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 364 : backward pass time 10.14 +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 364 : optimizer step time 1.58 +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 364-------------------- +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:28:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439530 41.514660 41.386032 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.609375 107.603516 +util 65.750000 70.000000 61.000000 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 365 : avg iteration time 160.00 seconds +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 365 : data loading time 0.00 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 365 : avg forward pass time 71.42 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 365 : backward pass time 8.40 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 365 : optimizer step time 0.40 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 365-------------------- +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6670 iter count: 1.0 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6670 iter count: 1.0 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6670 iter count: 1.0 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6670iter count: 1.0 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:31:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439533 41.514687 41.386032 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.640625 67.707031 67.441406 +util 94.000000 99.000000 88.000000 +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 366 : avg iteration time 46.98 seconds +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 366 : data loading time 0.00 +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 366 : avg forward pass time 12.56 +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 366 : backward pass time 10.14 +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 366 : optimizer step time 1.57 +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 366-------------------- +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:32:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439674 41.515179 41.386082 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.723633 67.804688 67.480469 +util 93.750000 100.000000 88.000000 +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 367 : avg iteration time 45.85 seconds +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 367 : data loading time 0.00 +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 367 : avg forward pass time 12.00 +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 367 : backward pass time 10.13 +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 367 : optimizer step time 1.57 +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 367-------------------- +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:33:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439669 41.515179 41.386082 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.645508 67.707031 67.460938 +util 93.250000 98.000000 89.000000 +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 368 : avg iteration time 48.43 seconds +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 368 : data loading time 0.00 +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 368 : avg forward pass time 13.29 +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 368 : backward pass time 10.13 +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 368 : optimizer step time 1.57 +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 368-------------------- +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:33:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439669 41.515179 41.386082 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.655273 67.687500 67.558594 +util 91.750000 96.000000 88.000000 +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 369 : avg iteration time 48.59 seconds +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 369 : data loading time 0.00 +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 369 : avg forward pass time 13.37 +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 369 : backward pass time 10.14 +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 369 : optimizer step time 1.57 +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 369-------------------- +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:34:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439829 41.515778 41.386101 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.726562 107.564453 +util 60.500000 65.000000 56.000000 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 370 : avg iteration time 162.80 seconds +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 370 : data loading time 0.00 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 370 : avg forward pass time 72.81 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 370 : backward pass time 8.40 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 370 : optimizer step time 0.39 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 370-------------------- +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6250 iter count: 1.0 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6250 iter count: 1.0 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6250 iter count: 1.0 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6250iter count: 1.0 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:37:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439868 41.515930 41.386105 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.744629 67.787109 67.617188 +util 92.000000 96.000000 88.000000 +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 371 : avg iteration time 44.40 seconds +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 371 : data loading time 0.00 +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 371 : avg forward pass time 11.27 +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 371 : backward pass time 10.15 +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 371 : optimizer step time 1.57 +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 371-------------------- +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:38:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439918 41.516151 41.386105 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.686035 67.708984 67.617188 +util 96.500000 99.000000 93.000000 +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 372 : avg iteration time 43.26 seconds +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 372 : data loading time 0.00 +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 372 : avg forward pass time 10.68 +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 372 : backward pass time 10.17 +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 372 : optimizer step time 1.57 +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 372-------------------- +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:38:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439925 41.516182 41.386105 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.708984 67.656250 +util 97.250000 100.000000 94.000000 +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 373 : avg iteration time 48.61 seconds +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 373 : data loading time 0.00 +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 373 : avg forward pass time 13.38 +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 373 : backward pass time 10.15 +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 373 : optimizer step time 1.58 +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 373-------------------- +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:39:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439961 41.516327 41.386105 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.700684 67.708984 67.675781 +util 92.500000 97.000000 88.000000 +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 374 : avg iteration time 45.93 seconds +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 374 : data loading time 0.00 +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 374 : avg forward pass time 12.04 +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 374 : backward pass time 10.14 +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 374 : optimizer step time 1.57 +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 374-------------------- +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:40:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439663 41.515087 41.386112 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.712402 107.740234 107.628906 +util 62.250000 67.000000 57.000000 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 375 : avg iteration time 160.42 seconds +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 375 : data loading time 0.00 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 375 : avg forward pass time 71.63 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 375 : backward pass time 8.40 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 375 : optimizer step time 0.40 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 375-------------------- +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6445 iter count: 1.0 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6445 iter count: 1.0 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6445 iter count: 1.0 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6445iter count: 1.0 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439663 41.515079 41.386105 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.651855 67.728516 67.421875 +util 92.250000 97.000000 88.000000 +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 376 : avg iteration time 46.98 seconds +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 376 : data loading time 0.00 +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 376 : avg forward pass time 12.56 +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 376 : backward pass time 10.14 +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 376 : optimizer step time 1.57 +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 376-------------------- +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:43:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439713 41.515282 41.386131 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.671387 67.708984 67.558594 +util 96.500000 99.000000 93.000000 +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 377 : avg iteration time 45.86 seconds +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 377 : data loading time 0.00 +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 377 : avg forward pass time 12.00 +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 377 : backward pass time 10.15 +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 377 : optimizer step time 1.55 +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 377-------------------- +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:44:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439908 41.516033 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.710449 67.787109 67.480469 +util 93.750000 99.000000 88.000000 +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 378 : avg iteration time 45.86 seconds +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 378 : data loading time 0.00 +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 378 : avg forward pass time 12.02 +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 378 : backward pass time 10.11 +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 378 : optimizer step time 1.57 +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 378-------------------- +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:45:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440025 41.516518 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.646973 67.728516 67.402344 +util 96.500000 99.000000 93.000000 +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 379 : avg iteration time 45.93 seconds +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 379 : data loading time 0.00 +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 379 : avg forward pass time 12.05 +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 379 : backward pass time 10.12 +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 379 : optimizer step time 1.57 +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 379-------------------- +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:46:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439652 41.515026 41.386143 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.628906 107.603516 +util 65.000000 70.000000 60.000000 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 380 : avg iteration time 160.20 seconds +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 380 : data loading time 0.00 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 380 : avg forward pass time 71.51 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 380 : backward pass time 8.40 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 380 : optimizer step time 0.40 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 380-------------------- +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6328 iter count: 1.0 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6328 iter count: 1.0 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6328 iter count: 1.0 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6328iter count: 1.0 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:48:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439652 41.515026 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.357422 67.421875 67.335938 +util 97.500000 99.000000 95.000000 +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 381 : avg iteration time 47.03 seconds +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 381 : data loading time 0.00 +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 381 : avg forward pass time 12.59 +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 381 : backward pass time 10.14 +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 381 : optimizer step time 1.58 +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 381-------------------- +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:49:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439773 41.515484 41.386166 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.420898 67.441406 67.414062 +util 94.250000 100.000000 89.000000 +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 382 : avg iteration time 48.50 seconds +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 382 : data loading time 0.00 +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 382 : avg forward pass time 13.32 +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 382 : backward pass time 10.16 +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 382 : optimizer step time 1.57 +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 382-------------------- +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:50:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439995 41.516396 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.450195 67.910156 67.296875 +util 94.250000 99.000000 88.000000 +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 383 : avg iteration time 43.34 seconds +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 383 : data loading time 0.00 +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 383 : avg forward pass time 10.73 +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 383 : backward pass time 10.16 +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 383 : optimizer step time 1.57 +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 383-------------------- +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:51:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440058 41.516624 41.386166 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.420898 67.558594 67.375000 +util 95.250000 99.000000 91.000000 +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 384 : avg iteration time 48.51 seconds +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 384 : data loading time 0.00 +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 384 : avg forward pass time 13.31 +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 384 : backward pass time 10.16 +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 384 : optimizer step time 1.57 +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 384-------------------- +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:51:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439686 41.515156 41.386143 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.683105 107.701172 107.628906 +util 68.500000 73.000000 63.000000 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 385 : avg iteration time 160.26 seconds +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 385 : data loading time 0.00 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 385 : avg forward pass time 71.53 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 385 : backward pass time 8.39 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 385 : optimizer step time 0.40 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 385-------------------- +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6035 iter count: 1.0 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6035 iter count: 1.0 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6035 iter count: 1.0 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6035iter count: 1.0 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:54:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439690 41.515144 41.386147 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.811523 67.882812 67.597656 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 386 : avg iteration time 49.46 seconds +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 386 : data loading time 0.00 +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 386 : avg forward pass time 13.81 +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 386 : backward pass time 10.13 +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 386 : optimizer step time 1.58 +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 386-------------------- +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:55:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439745 41.515388 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.728516 67.785156 67.558594 +util 93.500000 99.000000 88.000000 +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 387 : avg iteration time 45.82 seconds +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 387 : data loading time 0.00 +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 387 : avg forward pass time 11.99 +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 387 : backward pass time 10.14 +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 387 : optimizer step time 1.57 +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 387-------------------- +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:56:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440070 41.516674 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.899414 67.949219 67.882812 +util 93.750000 98.000000 89.000000 +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 388 : avg iteration time 45.84 seconds +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 388 : data loading time 0.00 +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 388 : avg forward pass time 11.99 +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 388 : backward pass time 10.14 +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 388 : optimizer step time 1.57 +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 388-------------------- +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:57:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440078 41.516674 41.386166 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.806641 67.882812 67.578125 +util 94.500000 99.000000 89.000000 +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 389 : avg iteration time 48.51 seconds +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 389 : data loading time 0.00 +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 389 : avg forward pass time 13.32 +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 389 : backward pass time 10.14 +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 389 : optimizer step time 1.58 +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 389-------------------- +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 03:57:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439575 41.514706 41.386143 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.687500 107.525391 +util 67.500000 74.000000 59.000000 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 390 : avg iteration time 165.40 seconds +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 390 : data loading time 0.00 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 390 : avg forward pass time 74.13 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 390 : backward pass time 8.38 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 390 : optimizer step time 0.40 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 390-------------------- +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6543 iter count: 1.0 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6543 iter count: 1.0 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6543 iter count: 1.0 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6543iter count: 1.0 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:00:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439590 41.514706 41.386154 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.298828 67.539062 67.218750 +util 94.250000 99.000000 90.000000 +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 391 : avg iteration time 47.09 seconds +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 391 : data loading time 0.00 +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 391 : avg forward pass time 12.62 +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 391 : backward pass time 10.15 +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 391 : optimizer step time 1.57 +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 391-------------------- +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:01:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439647 41.514942 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.519531 67.335938 +util 96.250000 100.000000 89.000000 +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 392 : avg iteration time 45.89 seconds +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 392 : data loading time 0.00 +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 392 : avg forward pass time 12.01 +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 392 : backward pass time 10.14 +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 392 : optimizer step time 1.58 +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 392-------------------- +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:02:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439956 41.516178 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.421875 67.375000 +util 94.250000 99.000000 89.000000 +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 393 : avg iteration time 45.88 seconds +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 393 : data loading time 0.00 +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 393 : avg forward pass time 12.02 +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 393 : backward pass time 10.14 +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 393 : optimizer step time 1.57 +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 393-------------------- +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:02:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440165 41.516907 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.352539 67.519531 67.296875 +util 93.750000 100.000000 88.000000 +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 394 : avg iteration time 43.19 seconds +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 394 : data loading time 0.00 +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 394 : avg forward pass time 10.68 +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 394 : backward pass time 10.13 +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 394 : optimizer step time 1.57 +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 394-------------------- +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:03:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440068 41.516499 41.386166 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.648438 107.544922 +util 65.000000 70.000000 60.000000 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 395 : avg iteration time 154.87 seconds +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 395 : data loading time 0.00 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 395 : avg forward pass time 68.85 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 395 : backward pass time 8.39 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 395 : optimizer step time 0.40 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 395-------------------- +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5674 iter count: 1.0 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5674 iter count: 1.0 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5674 iter count: 1.0 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5674iter count: 1.0 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:06:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439964 41.516094 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.539062 67.316406 +util 95.250000 100.000000 90.000000 +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 396 : avg iteration time 44.30 seconds +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 396 : data loading time 0.00 +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 396 : avg forward pass time 11.23 +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 396 : backward pass time 10.13 +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 396 : optimizer step time 1.57 +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 396-------------------- +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:06:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439962 41.516094 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.519531 67.375000 +util 97.750000 100.000000 95.000000 +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 397 : avg iteration time 45.94 seconds +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 397 : data loading time 0.00 +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 397 : avg forward pass time 12.02 +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 397 : backward pass time 10.16 +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 397 : optimizer step time 1.57 +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 397-------------------- +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:07:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440027 41.516335 41.386143 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.480469 67.355469 +util 96.000000 100.000000 91.000000 +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 398 : avg iteration time 48.47 seconds +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 398 : data loading time 0.00 +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 398 : avg forward pass time 13.31 +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 398 : backward pass time 10.15 +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 398 : optimizer step time 1.58 +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 398-------------------- +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:08:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440174 41.516846 41.386238 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.313477 67.363281 67.296875 +util 96.000000 100.000000 91.000000 +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 399 : avg iteration time 46.00 seconds +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 399 : data loading time 0.00 +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 399 : avg forward pass time 12.07 +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 399 : backward pass time 10.15 +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 399 : optimizer step time 1.58 +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 399-------------------- +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:09:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440070 41.516407 41.386261 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.697754 107.746094 107.681641 +util 60.750000 65.000000 56.000000 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 400 : avg iteration time 155.04 seconds +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 400 : data loading time 0.00 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 400 : avg forward pass time 68.95 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 400 : backward pass time 8.39 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 400 : optimizer step time 0.40 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 400-------------------- +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6016 iter count: 1.0 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6016 iter count: 1.0 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6016 iter count: 1.0 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6016iter count: 1.0 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 04:11:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 04:12:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.42 seconds +[May 4, 2026 - 04:12:05 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 400 +[May 4, 2026 - 04:12:05 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000400 +[May 4, 2026 - 04:12:52 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 400 +[May 4, 2026 - 04:12:52 | INFO | fastgen.trainer:validate:358 ] Validation iteration 400 +[May 4, 2026 - 04:12:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:12:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 4, 2026 - 04:12:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.498 +[May 4, 2026 - 04:12:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 04:13:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.802 +[May 4, 2026 - 04:13:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 04:13:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.085 +[May 4, 2026 - 04:13:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 04:13:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.965 +[May 4, 2026 - 04:13:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 04:13:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.559 +[May 4, 2026 - 04:13:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 04:13:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.722 +[May 4, 2026 - 04:13:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 04:13:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.548 +[May 4, 2026 - 04:13:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 04:13:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.138 +[May 4, 2026 - 04:13:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 04:13:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.148 +[May 4, 2026 - 04:13:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 04:13:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 04:13:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.311 +[May 4, 2026 - 04:13:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 400-------------------- +[May 4, 2026 - 04:13:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 04:13:59 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 400 +[May 4, 2026 - 04:13:59 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.478 (n=10) +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441445 41.521473 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.373047 68.730469 68.253906 +util 94.250000 99.000000 88.000000 +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 401 : avg iteration time 174.13 seconds +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 401 : data loading time 0.00 +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 401 : avg forward pass time 11.68 +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 401 : backward pass time 10.13 +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 401 : optimizer step time 1.58 +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 401-------------------- +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:14:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441450 41.521492 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.470703 69.121094 68.253906 +util 95.750000 100.000000 91.000000 +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 402 : avg iteration time 43.42 seconds +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 402 : data loading time 0.00 +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 402 : avg forward pass time 10.75 +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 402 : backward pass time 10.18 +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 402 : optimizer step time 1.57 +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 402-------------------- +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:15:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441451 41.521469 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.373047 68.730469 68.253906 +util 96.000000 99.000000 89.000000 +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 403 : avg iteration time 46.17 seconds +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 403 : data loading time 0.00 +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 403 : avg forward pass time 12.10 +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 403 : backward pass time 10.19 +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 403 : optimizer step time 1.59 +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 403-------------------- +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:16:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441444 41.521469 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.368164 68.652344 68.273438 +util 93.500000 100.000000 88.000000 +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 404 : avg iteration time 46.04 seconds +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 404 : data loading time 0.00 +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 404 : avg forward pass time 12.07 +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 404 : backward pass time 10.15 +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 404 : optimizer step time 1.57 +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 404-------------------- +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:17:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440228 41.516582 41.386337 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.662109 107.628906 +util 64.500000 71.000000 58.000000 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 405 : avg iteration time 162.98 seconds +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 405 : data loading time 0.00 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 405 : avg forward pass time 72.91 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 405 : backward pass time 8.38 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 405 : optimizer step time 0.40 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 405-------------------- +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6221 iter count: 1.0 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6221 iter count: 1.0 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6221 iter count: 1.0 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6221iter count: 1.0 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:19:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440226 41.516594 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.713867 67.843750 67.324219 +util 96.000000 99.000000 92.000000 +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 406 : avg iteration time 46.90 seconds +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 406 : data loading time 0.00 +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 406 : avg forward pass time 12.52 +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 406 : backward pass time 10.13 +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 406 : optimizer step time 1.58 +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 406-------------------- +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:20:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440288 41.516781 41.386326 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.650391 67.765625 67.304688 +util 95.750000 100.000000 88.000000 +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 407 : avg iteration time 45.85 seconds +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 407 : data loading time 0.00 +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 407 : avg forward pass time 11.99 +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 407 : backward pass time 10.15 +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 407 : optimizer step time 1.58 +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 407-------------------- +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:21:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440591 41.517986 41.386337 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.586914 67.667969 67.343750 +util 95.000000 99.000000 90.000000 +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 408 : avg iteration time 43.24 seconds +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 408 : data loading time 0.00 +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 408 : avg forward pass time 10.69 +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 408 : backward pass time 10.14 +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 408 : optimizer step time 1.57 +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 408-------------------- +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:21:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440580 41.517986 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.577148 67.667969 67.304688 +util 93.250000 99.000000 88.000000 +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 409 : avg iteration time 45.83 seconds +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 409 : data loading time 0.00 +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 409 : avg forward pass time 12.00 +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 409 : backward pass time 10.13 +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 409 : optimizer step time 1.57 +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 409-------------------- +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:22:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440160 41.516304 41.386314 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.726562 107.603516 +util 67.750000 72.000000 62.000000 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 410 : avg iteration time 160.31 seconds +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 410 : data loading time 0.00 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 410 : avg forward pass time 71.57 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 410 : backward pass time 8.41 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 410 : optimizer step time 0.40 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 410-------------------- +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5479 iter count: 1.0 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5479 iter count: 1.0 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5479 iter count: 1.0 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5479iter count: 1.0 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:25:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440161 41.516304 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.882812 67.363281 +util 94.250000 99.000000 89.000000 +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 411 : avg iteration time 49.54 seconds +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 411 : data loading time 0.00 +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 411 : avg forward pass time 13.84 +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 411 : backward pass time 10.15 +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 411 : optimizer step time 1.57 +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 411-------------------- +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:26:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440466 41.517517 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 93.750000 99.000000 88.000000 +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 412 : avg iteration time 45.90 seconds +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 412 : data loading time 0.00 +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 412 : avg forward pass time 12.02 +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 412 : backward pass time 10.13 +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 412 : optimizer step time 1.57 +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 412-------------------- +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:27:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440738 41.518612 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 92.000000 97.000000 87.000000 +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 413 : avg iteration time 45.87 seconds +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 413 : data loading time 0.00 +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 413 : avg forward pass time 11.99 +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 413 : backward pass time 10.15 +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 413 : optimizer step time 1.57 +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 413-------------------- +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:27:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:28:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440805 41.518860 41.386337 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777344 67.902344 67.402344 +util 96.250000 100.000000 91.000000 +[May 4, 2026 - 04:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 414 : avg iteration time 45.97 seconds +[May 4, 2026 - 04:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 414 : data loading time 0.00 +[May 4, 2026 - 04:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 414 : avg forward pass time 12.04 +[May 4, 2026 - 04:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 414 : backward pass time 10.16 +[May 4, 2026 - 04:28:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 414 : optimizer step time 1.57 +[May 4, 2026 - 04:28:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 414-------------------- +[May 4, 2026 - 04:28:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 04:28:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 04:28:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:28:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440137 41.516190 41.386333 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.589844 107.544922 +util 67.500000 74.000000 62.000000 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 415 : avg iteration time 157.61 seconds +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 415 : data loading time 0.00 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 415 : avg forward pass time 70.23 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 415 : backward pass time 8.40 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 415 : optimizer step time 0.40 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 415-------------------- +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6074 iter count: 1.0 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6074 iter count: 1.0 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6074 iter count: 1.0 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6074iter count: 1.0 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:31:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440134 41.516197 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.733398 67.843750 67.402344 +util 93.500000 100.000000 88.000000 +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 416 : avg iteration time 47.00 seconds +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 416 : data loading time 0.00 +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 416 : avg forward pass time 12.55 +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 416 : backward pass time 10.16 +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 416 : optimizer step time 1.58 +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 416-------------------- +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:31:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440195 41.516430 41.386314 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.801758 67.941406 67.382812 +util 96.250000 99.000000 92.000000 +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 417 : avg iteration time 43.28 seconds +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 417 : data loading time 0.00 +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 417 : avg forward pass time 10.71 +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 417 : backward pass time 10.13 +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 417 : optimizer step time 1.57 +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 417-------------------- +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:32:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440549 41.517838 41.386322 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.882812 67.382812 +util 95.500000 99.000000 91.000000 +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 418 : avg iteration time 48.59 seconds +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 418 : data loading time 0.00 +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 418 : avg forward pass time 13.37 +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 418 : backward pass time 10.14 +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 418 : optimizer step time 1.57 +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 418-------------------- +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:33:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440377 41.517147 41.386318 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.882812 67.382812 +util 96.250000 99.000000 92.000000 +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 419 : avg iteration time 48.82 seconds +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 419 : data loading time 0.00 +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 419 : avg forward pass time 13.39 +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 419 : backward pass time 10.15 +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 419 : optimizer step time 1.57 +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 419-------------------- +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:34:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439775 41.514698 41.386318 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.658691 107.662109 107.648438 +util 65.750000 72.000000 58.000000 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 420 : avg iteration time 162.66 seconds +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 420 : data loading time 0.00 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 420 : avg forward pass time 72.74 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 420 : backward pass time 8.41 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 420 : optimizer step time 0.40 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 420-------------------- +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:37:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439768 41.514709 41.386318 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 67.863281 67.402344 +util 94.500000 99.000000 89.000000 +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 421 : avg iteration time 49.47 seconds +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 421 : data loading time 0.00 +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 421 : avg forward pass time 13.80 +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 421 : backward pass time 10.14 +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 421 : optimizer step time 1.56 +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 421-------------------- +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439943 41.515419 41.386318 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777344 67.902344 67.402344 +util 94.500000 99.000000 89.000000 +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 422 : avg iteration time 43.25 seconds +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 422 : data loading time 0.00 +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 422 : avg forward pass time 10.69 +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 422 : backward pass time 10.15 +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 422 : optimizer step time 1.58 +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 422-------------------- +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:38:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440187 41.516396 41.386318 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 91.750000 96.000000 87.000000 +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 423 : avg iteration time 45.97 seconds +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 423 : data loading time 0.00 +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 423 : avg forward pass time 12.06 +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 423 : backward pass time 10.14 +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 423 : optimizer step time 1.57 +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 423-------------------- +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:39:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440189 41.516373 41.386326 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 93.750000 98.000000 89.000000 +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 424 : avg iteration time 45.82 seconds +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 424 : data loading time 0.00 +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 424 : avg forward pass time 11.99 +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 424 : backward pass time 10.15 +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 424 : optimizer step time 1.57 +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 424-------------------- +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:40:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440048 41.515839 41.386318 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.697754 107.720703 107.628906 +util 67.000000 74.000000 59.000000 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 425 : avg iteration time 154.90 seconds +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 425 : data loading time 0.00 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 425 : avg forward pass time 68.87 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 425 : backward pass time 8.40 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 425 : optimizer step time 0.40 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 425-------------------- +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5410 iter count: 1.0 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5410 iter count: 1.0 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5410 iter count: 1.0 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5410iter count: 1.0 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:42:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440061 41.515846 41.386360 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.547852 67.570312 67.480469 +util 95.500000 99.000000 91.000000 +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 426 : avg iteration time 49.43 seconds +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 426 : data loading time 0.00 +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 426 : avg forward pass time 13.81 +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 426 : backward pass time 10.14 +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 426 : optimizer step time 1.56 +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 426-------------------- +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:43:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440124 41.516094 41.386360 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.547852 67.589844 67.421875 +util 97.500000 100.000000 94.000000 +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 427 : avg iteration time 48.52 seconds +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 427 : data loading time 0.00 +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 427 : avg forward pass time 13.32 +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 427 : backward pass time 10.14 +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 427 : optimizer step time 1.59 +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 427-------------------- +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:44:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440260 41.516644 41.386360 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.533203 67.570312 67.421875 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 428 : avg iteration time 45.87 seconds +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 428 : data loading time 0.00 +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 428 : avg forward pass time 12.01 +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 428 : backward pass time 10.14 +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 428 : optimizer step time 1.57 +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 428-------------------- +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:45:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440323 41.516895 41.386360 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.533203 67.570312 67.421875 +util 95.500000 99.000000 91.000000 +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 429 : avg iteration time 45.89 seconds +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 429 : data loading time 0.00 +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 429 : avg forward pass time 12.00 +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 429 : backward pass time 10.14 +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 429 : optimizer step time 1.58 +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 429-------------------- +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:45:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439882 41.515022 41.386448 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.751465 107.785156 107.740234 +util 64.500000 69.000000 59.000000 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 430 : avg iteration time 162.80 seconds +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 430 : data loading time 0.00 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 430 : avg forward pass time 72.82 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 430 : backward pass time 8.40 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 430 : optimizer step time 0.40 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 430-------------------- +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6104 iter count: 1.0 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6104 iter count: 1.0 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6104 iter count: 1.0 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6104iter count: 1.0 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:48:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439894 41.515049 41.386463 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.870117 68.019531 67.421875 +util 95.500000 99.000000 91.000000 +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 431 : avg iteration time 46.91 seconds +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 431 : data loading time 0.00 +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 431 : avg forward pass time 12.52 +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 431 : backward pass time 10.15 +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 431 : optimizer step time 1.58 +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 431-------------------- +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:49:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439947 41.515285 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 97.750000 100.000000 94.000000 +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 432 : avg iteration time 45.92 seconds +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 432 : data loading time 0.00 +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 432 : avg forward pass time 12.03 +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 432 : backward pass time 10.15 +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 432 : optimizer step time 1.59 +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 432-------------------- +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:50:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440185 41.516235 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.882812 67.421875 +util 97.500000 99.000000 94.000000 +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 433 : avg iteration time 43.20 seconds +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 433 : data loading time 0.00 +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 433 : avg forward pass time 10.67 +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 433 : backward pass time 10.14 +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 433 : optimizer step time 1.57 +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 433-------------------- +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440189 41.516239 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.882812 67.421875 +util 94.750000 100.000000 89.000000 +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 434 : avg iteration time 45.90 seconds +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 434 : data loading time 0.00 +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 434 : avg forward pass time 12.02 +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 434 : backward pass time 10.15 +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 434 : optimizer step time 1.58 +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 434-------------------- +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:51:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440304 41.516689 41.386452 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.746094 107.623047 +util 67.000000 72.000000 61.000000 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 435 : avg iteration time 160.29 seconds +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 435 : data loading time 0.00 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 435 : avg forward pass time 71.57 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 435 : backward pass time 8.38 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 435 : optimizer step time 0.40 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 435-------------------- +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5879 iter count: 1.0 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5879 iter count: 1.0 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5879 iter count: 1.0 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5879iter count: 1.0 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:54:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440156 41.516083 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.772461 67.863281 67.500000 +util 95.500000 100.000000 90.000000 +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 436 : avg iteration time 46.88 seconds +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 436 : data loading time 0.00 +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 436 : avg forward pass time 12.51 +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 436 : backward pass time 10.15 +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 436 : optimizer step time 1.57 +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 436-------------------- +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:55:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440145 41.516060 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.882812 67.421875 +util 97.000000 99.000000 93.000000 +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 437 : avg iteration time 48.52 seconds +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 437 : data loading time 0.00 +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 437 : avg forward pass time 13.31 +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 437 : backward pass time 10.15 +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 437 : optimizer step time 1.58 +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 437-------------------- +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:55:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440150 41.516060 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.787109 67.902344 67.441406 +util 94.500000 99.000000 89.000000 +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 438 : avg iteration time 43.28 seconds +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 438 : data loading time 0.00 +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 438 : avg forward pass time 10.70 +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 438 : backward pass time 10.13 +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 438 : optimizer step time 1.59 +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 438-------------------- +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:56:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440205 41.516300 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.801758 67.902344 67.500000 +util 94.500000 100.000000 89.000000 +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 439 : avg iteration time 45.99 seconds +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 439 : data loading time 0.00 +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 439 : avg forward pass time 12.07 +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 439 : backward pass time 10.14 +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 439 : optimizer step time 1.58 +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 439-------------------- +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 04:57:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440205 41.516300 41.386452 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.642578 107.589844 +util 66.000000 73.000000 61.000000 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 440 : avg iteration time 165.23 seconds +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 440 : data loading time 0.00 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 440 : avg forward pass time 74.04 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 440 : backward pass time 8.41 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 440 : optimizer step time 0.40 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 440-------------------- +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6221 iter count: 1.0 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6221 iter count: 1.0 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6221 iter count: 1.0 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6221iter count: 1.0 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:00:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440135 41.515976 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.441406 67.335938 +util 97.000000 99.000000 93.000000 +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 441 : avg iteration time 44.32 seconds +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 441 : data loading time 0.00 +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 441 : avg forward pass time 11.22 +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 441 : backward pass time 10.15 +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 441 : optimizer step time 1.58 +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 441-------------------- +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:00:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440127 41.515976 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.264648 67.460938 67.199219 +util 93.500000 100.000000 88.000000 +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 442 : avg iteration time 45.90 seconds +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 442 : data loading time 0.00 +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 442 : avg forward pass time 12.02 +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 442 : backward pass time 10.14 +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 442 : optimizer step time 1.57 +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 442-------------------- +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:01:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440130 41.515980 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.863281 67.460938 +util 96.000000 99.000000 91.000000 +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 443 : avg iteration time 43.28 seconds +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 443 : data loading time 0.00 +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 443 : avg forward pass time 10.70 +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 443 : backward pass time 10.14 +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 443 : optimizer step time 1.58 +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 443-------------------- +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:02:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440421 41.517166 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.460938 67.355469 +util 94.500000 99.000000 89.000000 +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 444 : avg iteration time 45.97 seconds +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 444 : data loading time 0.00 +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 444 : avg forward pass time 12.04 +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 444 : backward pass time 10.16 +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 444 : optimizer step time 1.57 +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 444-------------------- +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:03:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440071 41.515755 41.386459 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.603516 107.589844 +util 64.500000 70.000000 59.000000 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 445 : avg iteration time 162.91 seconds +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 445 : data loading time 0.00 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 445 : avg forward pass time 72.87 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 445 : backward pass time 8.41 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 445 : optimizer step time 0.40 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 445-------------------- +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6260 iter count: 1.0 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6260 iter count: 1.0 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6260 iter count: 1.0 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6260iter count: 1.0 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:05:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440148 41.516022 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.882812 67.421875 +util 93.750000 99.000000 88.000000 +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 446 : avg iteration time 49.60 seconds +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 446 : data loading time 0.00 +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 446 : avg forward pass time 13.85 +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 446 : backward pass time 10.14 +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 446 : optimizer step time 1.59 +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 446-------------------- +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:06:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440146 41.516022 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.772461 67.882812 67.441406 +util 94.250000 100.000000 89.000000 +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 447 : avg iteration time 45.87 seconds +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 447 : data loading time 0.00 +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 447 : avg forward pass time 12.00 +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 447 : backward pass time 10.15 +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 447 : optimizer step time 1.57 +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 447-------------------- +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:07:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440149 41.516022 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.863281 67.421875 +util 97.500000 99.000000 94.000000 +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 448 : avg iteration time 48.49 seconds +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 448 : data loading time 0.00 +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 448 : avg forward pass time 13.31 +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 448 : backward pass time 10.13 +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 448 : optimizer step time 1.57 +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 448-------------------- +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:08:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440385 41.517006 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.787109 67.902344 67.441406 +util 97.500000 99.000000 94.000000 +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 449 : avg iteration time 45.80 seconds +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 449 : data loading time 0.00 +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 449 : avg forward pass time 11.97 +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 449 : backward pass time 10.15 +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 449 : optimizer step time 1.57 +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 449-------------------- +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:08:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440090 41.515781 41.386452 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.712402 107.740234 107.628906 +util 65.750000 73.000000 60.000000 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 450 : avg iteration time 160.29 seconds +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 450 : data loading time 0.00 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 450 : avg forward pass time 71.57 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 450 : backward pass time 8.38 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 450 : optimizer step time 0.40 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 450-------------------- +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6201 iter count: 1.0 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6201 iter count: 1.0 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6201 iter count: 1.0 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6201iter count: 1.0 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:11:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440081 41.515781 41.386459 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 67.863281 67.402344 +util 94.250000 99.000000 89.000000 +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 451 : avg iteration time 49.57 seconds +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 451 : data loading time 0.00 +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 451 : avg forward pass time 13.85 +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 451 : backward pass time 10.18 +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 451 : optimizer step time 1.57 +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 451-------------------- +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:12:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440206 41.516277 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.863281 67.421875 +util 94.750000 99.000000 90.000000 +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 452 : avg iteration time 45.85 seconds +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 452 : data loading time 0.00 +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 452 : avg forward pass time 12.00 +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 452 : backward pass time 10.14 +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 452 : optimizer step time 1.57 +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 452-------------------- +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:13:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440206 41.516277 41.386452 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.723633 67.863281 67.304688 +util 95.250000 99.000000 90.000000 +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 453 : avg iteration time 48.45 seconds +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 453 : data loading time 0.00 +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 453 : avg forward pass time 13.29 +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 453 : backward pass time 10.14 +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 453 : optimizer step time 1.58 +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 453-------------------- +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:14:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.631166 41.714901 41.576160 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.738281 67.882812 67.304688 +util 96.000000 100.000000 89.000000 +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 454 : avg iteration time 48.48 seconds +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 454 : data loading time 0.00 +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 454 : avg forward pass time 12.73 +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 454 : backward pass time 10.12 +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 454 : optimizer step time 1.58 +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 454-------------------- +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:14:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.631288 41.714630 41.576328 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.678223 107.701172 107.609375 +util 67.500000 72.000000 62.000000 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 455 : avg iteration time 157.57 seconds +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 455 : data loading time 0.00 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 455 : avg forward pass time 70.21 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 455 : backward pass time 8.39 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 455 : optimizer step time 0.39 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 455-------------------- +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5781 iter count: 1.0 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5781 iter count: 1.0 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5781 iter count: 1.0 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5781iter count: 1.0 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:17:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.631871 41.715527 41.576435 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.441406 67.316406 +util 93.000000 97.000000 88.000000 +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 456 : avg iteration time 46.93 seconds +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 456 : data loading time 0.00 +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 456 : avg forward pass time 12.53 +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 456 : backward pass time 10.14 +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 456 : optimizer step time 1.58 +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 456-------------------- +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:18:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.632196 41.715801 41.576633 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.352539 67.460938 67.316406 +util 95.000000 100.000000 90.000000 +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 457 : avg iteration time 48.58 seconds +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 457 : data loading time 0.00 +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 457 : avg forward pass time 13.36 +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 457 : backward pass time 10.17 +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 457 : optimizer step time 1.57 +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 457-------------------- +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:19:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.632846 41.717033 41.576771 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.259766 67.500000 67.179688 +util 94.500000 99.000000 89.000000 +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 458 : avg iteration time 48.63 seconds +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 458 : data loading time 0.00 +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 458 : avg forward pass time 13.37 +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 458 : backward pass time 10.16 +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 458 : optimizer step time 1.57 +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 458-------------------- +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:20:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.633175 41.717155 41.576843 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.264648 67.460938 67.199219 +util 93.750000 100.000000 88.000000 +[May 4, 2026 - 05:20:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 459 : avg iteration time 48.49 seconds +[May 4, 2026 - 05:20:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 459 : data loading time 0.00 +[May 4, 2026 - 05:20:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 459 : avg forward pass time 13.31 +[May 4, 2026 - 05:20:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 459 : backward pass time 10.13 +[May 4, 2026 - 05:20:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 459 : optimizer step time 1.58 +[May 4, 2026 - 05:20:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 459-------------------- +[May 4, 2026 - 05:20:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:20:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:20:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:20:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.633642 41.716923 41.577820 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.648438 107.642578 +util 62.750000 68.000000 55.000000 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 460 : avg iteration time 160.31 seconds +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 460 : data loading time 0.00 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 460 : avg forward pass time 71.56 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 460 : backward pass time 8.41 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 460 : optimizer step time 0.40 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 460-------------------- +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5449 iter count: 1.0 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5449 iter count: 1.0 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5449 iter count: 1.0 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5449iter count: 1.0 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:23:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.634264 41.717022 41.578094 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.925293 68.099609 67.402344 +util 98.000000 100.000000 94.000000 +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 461 : avg iteration time 44.46 seconds +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 461 : data loading time 0.00 +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 461 : avg forward pass time 11.29 +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 461 : backward pass time 10.15 +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 461 : optimizer step time 1.57 +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 461-------------------- +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:24:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.634708 41.717274 41.578369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.930176 68.099609 67.421875 +util 94.250000 100.000000 89.000000 +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 462 : avg iteration time 43.30 seconds +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 462 : data loading time 0.00 +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 462 : avg forward pass time 10.70 +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 462 : backward pass time 10.15 +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 462 : optimizer step time 1.59 +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 462-------------------- +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:24:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.635013 41.717289 41.578426 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.886230 68.099609 67.246094 +util 97.250000 100.000000 93.000000 +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 463 : avg iteration time 45.89 seconds +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 463 : data loading time 0.00 +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 463 : avg forward pass time 12.03 +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 463 : backward pass time 10.13 +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 463 : optimizer step time 1.56 +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 463-------------------- +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:25:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.635089 41.717316 41.578476 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.886230 68.099609 67.246094 +util 94.500000 100.000000 89.000000 +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 464 : avg iteration time 45.87 seconds +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 464 : data loading time 0.00 +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 464 : avg forward pass time 12.01 +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 464 : backward pass time 10.14 +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 464 : optimizer step time 1.58 +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 464-------------------- +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:26:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.634940 41.716526 41.578693 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.662109 107.589844 +util 63.500000 70.000000 58.000000 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 465 : avg iteration time 160.05 seconds +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 465 : data loading time 0.00 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 465 : avg forward pass time 71.45 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 465 : backward pass time 8.40 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 465 : optimizer step time 0.39 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 465-------------------- +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5083 iter count: 1.0 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5083 iter count: 1.0 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5083 iter count: 1.0 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5083iter count: 1.0 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:28:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.635150 41.716534 41.578949 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.743164 67.882812 67.324219 +util 96.500000 99.000000 92.000000 +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 466 : avg iteration time 44.28 seconds +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 466 : data loading time 0.00 +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 466 : avg forward pass time 11.20 +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 466 : backward pass time 10.16 +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 466 : optimizer step time 1.58 +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 466-------------------- +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.635691 41.717258 41.579807 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.694336 67.824219 67.304688 +util 95.500000 99.000000 91.000000 +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 467 : avg iteration time 48.49 seconds +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 467 : data loading time 0.00 +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 467 : avg forward pass time 13.32 +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 467 : backward pass time 10.14 +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 467 : optimizer step time 1.57 +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 467-------------------- +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:30:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.635948 41.717609 41.579922 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.733398 67.863281 67.343750 +util 96.000000 99.000000 91.000000 +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 468 : avg iteration time 48.50 seconds +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 468 : data loading time 0.00 +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 468 : avg forward pass time 13.31 +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 468 : backward pass time 10.15 +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 468 : optimizer step time 1.57 +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 468-------------------- +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:31:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636399 41.718403 41.580799 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.355469 67.324219 +util 94.250000 99.000000 89.000000 +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 469 : avg iteration time 45.88 seconds +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 469 : data loading time 0.00 +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 469 : avg forward pass time 12.01 +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 469 : backward pass time 10.13 +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 469 : optimizer step time 1.57 +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 469-------------------- +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:32:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636419 41.717529 41.581738 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.642578 107.609375 +util 65.250000 70.000000 59.000000 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 470 : avg iteration time 154.93 seconds +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 470 : data loading time 0.00 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 470 : avg forward pass time 68.88 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 470 : backward pass time 8.40 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 470 : optimizer step time 0.40 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 470-------------------- +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5713 iter count: 1.0 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5713 iter count: 1.0 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5713 iter count: 1.0 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5713iter count: 1.0 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:34:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636538 41.717518 41.581974 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.319824 67.324219 67.318359 +util 92.750000 99.000000 87.000000 +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 471 : avg iteration time 44.26 seconds +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 471 : data loading time 0.00 +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 471 : avg forward pass time 11.20 +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 471 : backward pass time 10.15 +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 471 : optimizer step time 1.57 +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 471-------------------- +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:35:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636701 41.717793 41.582165 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.354004 67.402344 67.337891 +util 94.500000 99.000000 89.000000 +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 472 : avg iteration time 48.51 seconds +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 472 : data loading time 0.00 +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 472 : avg forward pass time 13.33 +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 472 : backward pass time 10.14 +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 472 : optimizer step time 1.58 +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 472-------------------- +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:36:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637168 41.719486 41.582241 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.490723 67.494141 67.480469 +util 92.750000 99.000000 87.000000 +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 473 : avg iteration time 45.89 seconds +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 473 : data loading time 0.00 +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 473 : avg forward pass time 12.02 +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 473 : backward pass time 10.14 +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 473 : optimizer step time 1.57 +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 473-------------------- +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:37:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637406 41.720089 41.582474 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.261230 67.441406 67.201172 +util 95.750000 100.000000 91.000000 +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 474 : avg iteration time 48.48 seconds +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 474 : data loading time 0.00 +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 474 : avg forward pass time 13.31 +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 474 : backward pass time 10.14 +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 474 : optimizer step time 1.57 +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 474-------------------- +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:37:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637324 41.719749 41.582478 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.517090 107.609375 107.486328 +util 64.750000 72.000000 59.000000 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 475 : avg iteration time 155.15 seconds +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 475 : data loading time 0.00 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 475 : avg forward pass time 69.00 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 475 : backward pass time 8.39 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 475 : optimizer step time 0.40 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 475-------------------- +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5435 iter count: 1.0 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5435 iter count: 1.0 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5435 iter count: 1.0 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5435iter count: 1.0 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:40:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637115 41.718571 41.582478 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.148926 67.343750 67.083984 +util 96.000000 99.000000 91.000000 +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 476 : avg iteration time 46.91 seconds +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 476 : data loading time 0.00 +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 476 : avg forward pass time 12.54 +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 476 : backward pass time 10.12 +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 476 : optimizer step time 1.56 +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 476-------------------- +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:41:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637207 41.718643 41.582588 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.533203 67.304688 +util 95.500000 99.000000 91.000000 +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 477 : avg iteration time 45.90 seconds +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 477 : data loading time 0.00 +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 477 : avg forward pass time 12.01 +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 477 : backward pass time 10.14 +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 477 : optimizer step time 1.57 +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 477-------------------- +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637282 41.718929 41.582603 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.251465 67.298828 67.109375 +util 93.500000 99.000000 88.000000 +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 478 : avg iteration time 45.96 seconds +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 478 : data loading time 0.00 +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 478 : avg forward pass time 12.03 +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 478 : backward pass time 10.17 +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 478 : optimizer step time 1.57 +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 478-------------------- +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:42:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637362 41.718952 41.582756 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.515137 67.669922 67.050781 +util 96.250000 99.000000 91.000000 +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 479 : avg iteration time 48.54 seconds +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 479 : data loading time 0.00 +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 479 : avg forward pass time 13.32 +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 479 : backward pass time 10.16 +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 479 : optimizer step time 1.57 +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 479-------------------- +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637383 41.718929 41.582798 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.712402 107.720703 107.687500 +util 66.250000 73.000000 61.000000 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 480 : avg iteration time 160.11 seconds +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 480 : data loading time 0.00 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 480 : avg forward pass time 71.49 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 480 : backward pass time 8.37 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 480 : optimizer step time 0.40 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 480-------------------- +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5186 iter count: 1.0 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5186 iter count: 1.0 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5186 iter count: 1.0 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5186iter count: 1.0 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:46:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637323 41.718609 41.582798 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.334473 67.337891 67.324219 +util 97.000000 100.000000 92.000000 +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 481 : avg iteration time 46.92 seconds +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 481 : data loading time 0.00 +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 481 : avg forward pass time 12.52 +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 481 : backward pass time 10.16 +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 481 : optimizer step time 1.57 +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 481-------------------- +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:46:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637369 41.718651 41.582806 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.241699 67.298828 67.070312 +util 93.250000 100.000000 89.000000 +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 482 : avg iteration time 48.48 seconds +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 482 : data loading time 0.00 +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 482 : avg forward pass time 13.31 +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 482 : backward pass time 10.16 +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 482 : optimizer step time 1.57 +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 482-------------------- +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:47:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637474 41.718956 41.582813 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.339355 67.402344 67.318359 +util 97.000000 100.000000 92.000000 +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 483 : avg iteration time 48.50 seconds +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 483 : data loading time 0.00 +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 483 : avg forward pass time 13.33 +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 483 : backward pass time 10.13 +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 483 : optimizer step time 1.57 +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 483-------------------- +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:48:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637475 41.718960 41.582825 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.144043 67.441406 67.044922 +util 95.750000 100.000000 91.000000 +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 484 : avg iteration time 43.28 seconds +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 484 : data loading time 0.00 +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 484 : avg forward pass time 10.70 +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 484 : backward pass time 10.15 +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 484 : optimizer step time 1.57 +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 484-------------------- +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:49:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637552 41.718960 41.582920 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.658691 107.765625 107.623047 +util 67.500000 72.000000 61.000000 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 485 : avg iteration time 160.13 seconds +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 485 : data loading time 0.00 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 485 : avg forward pass time 71.49 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 485 : backward pass time 8.38 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 485 : optimizer step time 0.41 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 485-------------------- +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6289 iter count: 1.0 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6289 iter count: 1.0 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6289 iter count: 1.0 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6289iter count: 1.0 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:51:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637592 41.718983 41.583061 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.319824 67.382812 67.298828 +util 95.000000 99.000000 89.000000 +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 486 : avg iteration time 44.34 seconds +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 486 : data loading time 0.00 +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 486 : avg forward pass time 11.23 +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 486 : backward pass time 10.16 +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 486 : optimizer step time 1.57 +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 486-------------------- +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:52:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637659 41.719032 41.583176 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.832031 67.318359 +util 96.000000 99.000000 90.000000 +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 487 : avg iteration time 48.68 seconds +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 487 : data loading time 0.00 +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 487 : avg forward pass time 13.40 +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 487 : backward pass time 10.14 +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 487 : optimizer step time 1.58 +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 487-------------------- +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:53:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637766 41.719234 41.583256 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.832031 67.318359 +util 97.000000 100.000000 92.000000 +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 488 : avg iteration time 45.95 seconds +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 488 : data loading time 0.00 +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 488 : avg forward pass time 12.04 +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 488 : backward pass time 10.15 +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 488 : optimizer step time 1.57 +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 488-------------------- +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:54:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637971 41.720036 41.583256 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.441895 67.812500 67.318359 +util 95.750000 100.000000 91.000000 +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 489 : avg iteration time 45.93 seconds +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 489 : data loading time 0.00 +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 489 : avg forward pass time 12.03 +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 489 : backward pass time 10.15 +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 489 : optimizer step time 1.57 +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 489-------------------- +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:55:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637755 41.719025 41.583317 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.648438 107.603516 +util 65.500000 72.000000 61.000000 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 490 : avg iteration time 162.76 seconds +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 490 : data loading time 0.00 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 490 : avg forward pass time 72.80 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 490 : backward pass time 8.41 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 490 : optimizer step time 0.40 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 490-------------------- +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6074 iter count: 1.0 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6074 iter count: 1.0 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6074 iter count: 1.0 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6074iter count: 1.0 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:57:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637795 41.719002 41.583313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.375000 67.324219 +util 95.750000 99.000000 89.000000 +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 491 : avg iteration time 46.96 seconds +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 491 : data loading time 0.00 +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 491 : avg forward pass time 12.53 +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 491 : backward pass time 10.18 +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 491 : optimizer step time 1.58 +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 491-------------------- +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:58:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637865 41.718987 41.583324 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.367188 67.375000 67.343750 +util 95.500000 100.000000 91.000000 +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 492 : avg iteration time 45.88 seconds +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 492 : data loading time 0.00 +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 492 : avg forward pass time 12.02 +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 492 : backward pass time 10.14 +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 492 : optimizer step time 1.56 +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 492-------------------- +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637914 41.719017 41.583332 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.421875 67.375000 +util 95.500000 99.000000 89.000000 +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 493 : avg iteration time 45.91 seconds +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 493 : data loading time 0.00 +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 493 : avg forward pass time 12.02 +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 493 : backward pass time 10.15 +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 493 : optimizer step time 1.56 +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 493-------------------- +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:00:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638013 41.719257 41.583309 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.402344 67.375000 +util 95.250000 99.000000 89.000000 +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 494 : avg iteration time 45.93 seconds +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 494 : data loading time 0.00 +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 494 : avg forward pass time 12.02 +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 494 : backward pass time 10.15 +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 494 : optimizer step time 1.58 +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 494-------------------- +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:00:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637964 41.719059 41.583313 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.642578 107.628906 +util 65.000000 72.000000 60.000000 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 495 : avg iteration time 160.30 seconds +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 495 : data loading time 0.00 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 495 : avg forward pass time 71.57 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 495 : backward pass time 8.41 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 495 : optimizer step time 0.39 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 495-------------------- +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6299 iter count: 1.0 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6299 iter count: 1.0 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6299 iter count: 1.0 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6299iter count: 1.0 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:03:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637980 41.719059 41.583317 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.036621 67.304688 66.947266 +util 93.250000 100.000000 88.000000 +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 496 : avg iteration time 47.02 seconds +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 496 : data loading time 0.00 +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 496 : avg forward pass time 12.58 +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 496 : backward pass time 10.13 +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 496 : optimizer step time 1.58 +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 496-------------------- +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0140 iter count: 1.0 +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:04:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638087 41.719326 41.583317 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.031738 67.285156 66.947266 +util 93.000000 97.000000 87.000000 +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 497 : avg iteration time 43.25 seconds +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 497 : data loading time 0.00 +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 497 : avg forward pass time 10.69 +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 497 : backward pass time 10.15 +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 497 : optimizer step time 1.57 +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 497-------------------- +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:05:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638111 41.719303 41.583328 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.031738 67.285156 66.947266 +util 97.750000 100.000000 93.000000 +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 498 : avg iteration time 45.87 seconds +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 498 : data loading time 0.00 +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 498 : avg forward pass time 12.01 +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 498 : backward pass time 10.13 +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 498 : optimizer step time 1.57 +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 498-------------------- +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:05:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638143 41.719315 41.583347 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.031738 67.285156 66.947266 +util 95.000000 100.000000 90.000000 +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 499 : avg iteration time 45.86 seconds +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 499 : data loading time 0.00 +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 499 : avg forward pass time 12.00 +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 499 : backward pass time 10.15 +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 499 : optimizer step time 1.57 +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 499-------------------- +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:06:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638093 41.718903 41.583454 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.667969 107.583984 +util 62.500000 67.000000 56.000000 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 500 : avg iteration time 160.48 seconds +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 500 : data loading time 0.00 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 500 : avg forward pass time 71.50 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 500 : backward pass time 8.73 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 500 : optimizer step time 0.40 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 500-------------------- +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5664 iter count: 1.0 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5664 iter count: 1.0 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5664 iter count: 1.0 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5664iter count: 1.0 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 06:09:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 06:09:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.25 seconds +[May 4, 2026 - 06:09:28 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 500 +[May 4, 2026 - 06:09:28 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000500 +[May 4, 2026 - 06:10:15 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 500 +[May 4, 2026 - 06:10:15 | INFO | fastgen.trainer:validate:358 ] Validation iteration 500 +[May 4, 2026 - 06:10:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 4, 2026 - 06:10:20 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.048 +[May 4, 2026 - 06:10:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 06:10:26 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.031 +[May 4, 2026 - 06:10:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 06:10:30 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.436 +[May 4, 2026 - 06:10:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 06:10:35 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.620 +[May 4, 2026 - 06:10:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 06:10:40 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.860 +[May 4, 2026 - 06:10:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 06:10:45 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.627 +[May 4, 2026 - 06:10:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 06:10:50 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.915 +[May 4, 2026 - 06:10:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 06:10:55 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.932 +[May 4, 2026 - 06:10:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:10:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 06:11:00 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.595 +[May 4, 2026 - 06:11:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 06:11:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 06:11:05 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.717 +[May 4, 2026 - 06:11:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 500-------------------- +[May 4, 2026 - 06:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 06:11:22 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 500 +[May 4, 2026 - 06:11:22 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.178 (n=10) +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639665 41.724731 41.583710 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.314453 68.332031 68.261719 +util 95.000000 100.000000 90.000000 +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 501 : avg iteration time 176.12 seconds +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 0.00 +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 12.99 +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 10.15 +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 1.56 +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 501-------------------- +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:12:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639856 41.725151 41.583836 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.333984 68.398438 68.312500 +util 93.500000 100.000000 90.000000 +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 46.02 seconds +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 12.07 +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 10.17 +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 1.57 +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 502-------------------- +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:12:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639915 41.725174 41.583832 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.304688 68.312500 68.281250 +util 93.250000 99.000000 88.000000 +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 46.00 seconds +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 12.06 +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 10.16 +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 1.55 +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 503-------------------- +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:13:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639955 41.725292 41.583900 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.236328 68.417969 68.175781 +util 95.250000 99.000000 91.000000 +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 48.64 seconds +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 13.37 +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 10.16 +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 1.58 +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 504-------------------- +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:14:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638766 41.720348 41.583862 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.609375 107.583984 +util 65.250000 73.000000 60.000000 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 160.16 seconds +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 71.52 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 8.39 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.40 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 505-------------------- +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5205 iter count: 1.0 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5205 iter count: 1.0 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5205 iter count: 1.0 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5205iter count: 1.0 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:17:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638798 41.720394 41.583866 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.645508 67.726562 67.402344 +util 94.000000 99.000000 89.000000 +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 46.97 seconds +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 12.55 +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 10.15 +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 1.57 +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 506-------------------- +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:17:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638930 41.720627 41.583939 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.694336 67.765625 67.480469 +util 93.750000 98.000000 87.000000 +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 45.89 seconds +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 12.01 +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 10.15 +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 1.58 +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 507-------------------- +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:18:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639204 41.721756 41.583939 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.665039 67.726562 67.480469 +util 96.250000 99.000000 91.000000 +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 45.84 seconds +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 11.99 +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 10.14 +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 1.58 +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 508-------------------- +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:19:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639202 41.721756 41.583916 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.679688 67.765625 67.421875 +util 95.500000 100.000000 88.000000 +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 48.45 seconds +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 13.30 +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 10.12 +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 1.57 +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 509-------------------- +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:20:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638792 41.719933 41.583931 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.667969 107.623047 +util 69.500000 74.000000 62.000000 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 157.40 seconds +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 70.12 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 8.37 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.40 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 510-------------------- +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5527 iter count: 1.0 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5527 iter count: 1.0 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5527 iter count: 1.0 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5527iter count: 1.0 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:22:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638809 41.719948 41.583961 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.791992 67.812500 67.785156 +util 97.500000 100.000000 92.000000 +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 49.55 seconds +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 13.85 +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 10.15 +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 1.56 +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 511-------------------- +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:23:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639053 41.720970 41.583939 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.285156 68.312500 68.203125 +util 93.500000 98.000000 87.000000 +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 45.94 seconds +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 12.03 +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 10.15 +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 1.58 +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 512-------------------- +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:24:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639281 41.721699 41.583961 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.304688 68.332031 68.222656 +util 93.000000 99.000000 88.000000 +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 43.34 seconds +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 10.74 +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 10.16 +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 1.56 +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 513-------------------- +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:25:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639300 41.721699 41.584057 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.294922 68.332031 68.183594 +util 96.250000 99.000000 91.000000 +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 43.30 seconds +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 10.72 +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 10.15 +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 1.56 +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 514-------------------- +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638864 41.719898 41.584053 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.667969 107.583984 +util 62.500000 66.000000 60.000000 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 160.35 seconds +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 71.57 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 8.40 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.41 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 515-------------------- +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6016 iter count: 1.0 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6016 iter count: 1.0 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6016 iter count: 1.0 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6016iter count: 1.0 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:28:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638926 41.719906 41.584099 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.876465 67.910156 67.865234 +util 95.500000 99.000000 89.000000 +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 49.62 seconds +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 13.88 +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 10.14 +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 1.58 +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 516-------------------- +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:29:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638959 41.719948 41.584187 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.866699 67.871094 67.865234 +util 93.500000 99.000000 89.000000 +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 43.33 seconds +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 10.72 +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 10.16 +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 1.58 +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 517-------------------- +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:30:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639340 41.721432 41.584187 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.842285 67.845703 67.832031 +util 95.000000 99.000000 88.000000 +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 43.33 seconds +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 10.72 +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 10.15 +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 1.58 +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 518-------------------- +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:30:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639338 41.721432 41.584198 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.876465 67.910156 67.865234 +util 93.250000 99.000000 90.000000 +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 45.85 seconds +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 12.00 +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 10.15 +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 1.58 +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 519-------------------- +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:31:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638421 41.717609 41.584198 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.603516 107.589844 +util 67.750000 72.000000 61.000000 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 165.34 seconds +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 74.08 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 8.41 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.40 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 520-------------------- +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5615 iter count: 1.0 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5615 iter count: 1.0 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5615 iter count: 1.0 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5615iter count: 1.0 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:34:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638504 41.717609 41.584209 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.817871 67.845703 67.734375 +util 96.000000 99.000000 90.000000 +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 46.96 seconds +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 12.54 +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 10.17 +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 1.58 +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 521-------------------- +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:35:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638605 41.717880 41.584320 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.847168 67.910156 67.826172 +util 92.500000 99.000000 87.000000 +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 48.65 seconds +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.00 +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 13.38 +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 10.14 +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 1.58 +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 522-------------------- +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:36:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638991 41.719353 41.584335 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.856934 67.949219 67.826172 +util 95.000000 100.000000 90.000000 +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 43.29 seconds +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 10.71 +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 10.16 +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 1.57 +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 523-------------------- +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:36:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639083 41.719608 41.584446 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.837402 67.871094 67.826172 +util 96.000000 99.000000 90.000000 +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 524 : avg iteration time 48.66 seconds +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 524 : data loading time 0.00 +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 524 : avg forward pass time 13.40 +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 524 : backward pass time 10.15 +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 524 : optimizer step time 1.57 +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 524-------------------- +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:37:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638643 41.717815 41.584476 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.531738 107.609375 107.505859 +util 63.750000 69.000000 59.000000 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 525 : avg iteration time 157.92 seconds +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 525 : data loading time 0.00 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 525 : avg forward pass time 70.37 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 525 : backward pass time 8.41 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 525 : optimizer step time 0.40 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 525-------------------- +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5264 iter count: 1.0 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5264 iter count: 1.0 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5264 iter count: 1.0 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5264iter count: 1.0 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:40:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638643 41.717800 41.584476 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.788574 67.792969 67.787109 +util 94.750000 99.000000 88.000000 +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 526 : avg iteration time 44.44 seconds +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 526 : data loading time 0.00 +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 526 : avg forward pass time 11.28 +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 526 : backward pass time 10.15 +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 526 : optimizer step time 1.58 +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 526-------------------- +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:40:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638789 41.718327 41.584499 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.247559 68.412109 67.753906 +util 93.500000 99.000000 89.000000 +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 527 : avg iteration time 48.54 seconds +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 527 : data loading time 0.00 +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 527 : avg forward pass time 13.32 +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 527 : backward pass time 10.16 +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 527 : optimizer step time 1.58 +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 527-------------------- +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:41:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639171 41.719799 41.584507 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.276855 68.412109 67.871094 +util 95.750000 100.000000 91.000000 +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 528 : avg iteration time 43.24 seconds +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 528 : data loading time 0.00 +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 528 : avg forward pass time 10.68 +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 528 : backward pass time 10.15 +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 528 : optimizer step time 1.58 +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 528-------------------- +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:42:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639191 41.719822 41.584484 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.257324 68.412109 67.792969 +util 94.250000 99.000000 90.000000 +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 529 : avg iteration time 48.54 seconds +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 529 : data loading time 0.00 +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 529 : avg forward pass time 13.33 +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 529 : backward pass time 10.16 +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 529 : optimizer step time 1.58 +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 529-------------------- +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:43:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639107 41.719406 41.584507 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.681641 107.570312 +util 65.250000 73.000000 60.000000 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 530 : avg iteration time 157.41 seconds +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 530 : data loading time 0.00 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 530 : avg forward pass time 70.13 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 530 : backward pass time 8.38 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 530 : optimizer step time 0.40 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 530-------------------- +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6113 iter count: 1.0 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6113 iter count: 1.0 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6113 iter count: 1.0 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6113iter count: 1.0 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:45:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639023 41.719112 41.584484 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.773438 67.376953 +util 93.500000 99.000000 89.000000 +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 531 : avg iteration time 49.61 seconds +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 531 : data loading time 0.00 +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 531 : avg forward pass time 13.88 +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 531 : backward pass time 10.15 +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 531 : optimizer step time 1.56 +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 531-------------------- +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:46:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639031 41.719101 41.584507 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358887 67.773438 67.220703 +util 97.750000 99.000000 94.000000 +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 532 : avg iteration time 48.53 seconds +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 532 : data loading time 0.00 +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 532 : avg forward pass time 13.34 +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 532 : backward pass time 10.14 +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 532 : optimizer step time 1.57 +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 532-------------------- +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:47:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639030 41.719112 41.584499 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.520020 67.890625 67.396484 +util 96.000000 99.000000 90.000000 +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 533 : avg iteration time 45.95 seconds +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 533 : data loading time 0.00 +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 533 : avg forward pass time 12.03 +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 533 : backward pass time 10.15 +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 533 : optimizer step time 1.58 +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 533-------------------- +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:48:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639029 41.719112 41.584492 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.534668 67.890625 67.416016 +util 97.250000 100.000000 92.000000 +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 534 : avg iteration time 45.82 seconds +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 534 : data loading time 0.00 +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 534 : avg forward pass time 11.99 +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 534 : backward pass time 10.13 +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 534 : optimizer step time 1.58 +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 534-------------------- +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:49:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639036 41.719135 41.584492 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.648438 107.623047 +util 63.000000 68.000000 58.000000 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 535 : avg iteration time 157.50 seconds +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 535 : data loading time 0.00 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 535 : avg forward pass time 70.16 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 535 : backward pass time 8.41 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 535 : optimizer step time 0.40 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 535-------------------- +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5508 iter count: 1.0 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5508 iter count: 1.0 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5508 iter count: 1.0 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5508iter count: 1.0 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:51:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639030 41.719021 41.584515 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.289062 67.296875 67.265625 +util 96.750000 100.000000 91.000000 +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 536 : avg iteration time 46.89 seconds +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 536 : data loading time 0.00 +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 536 : avg forward pass time 12.51 +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 536 : backward pass time 10.14 +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 536 : optimizer step time 1.58 +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 536-------------------- +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:52:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639030 41.719021 41.584515 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.313477 67.335938 67.246094 +util 97.000000 99.000000 92.000000 +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 537 : avg iteration time 48.41 seconds +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 537 : data loading time 0.00 +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 537 : avg forward pass time 13.28 +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 537 : backward pass time 10.13 +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 537 : optimizer step time 1.56 +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 537-------------------- +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:53:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639040 41.719055 41.584496 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.323242 67.355469 67.226562 +util 96.000000 99.000000 90.000000 +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 538 : avg iteration time 45.98 seconds +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 538 : data loading time 0.00 +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 538 : avg forward pass time 12.07 +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 538 : backward pass time 10.15 +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 538 : optimizer step time 1.57 +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 538-------------------- +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:54:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639032 41.719059 41.584518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.433594 67.246094 +util 96.500000 99.000000 91.000000 +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 539 : avg iteration time 48.61 seconds +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 539 : data loading time 0.00 +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 539 : avg forward pass time 13.37 +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 539 : backward pass time 10.14 +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 539 : optimizer step time 1.56 +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 539-------------------- +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:54:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638932 41.718498 41.584621 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.628906 107.525391 +util 66.250000 74.000000 58.000000 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 540 : avg iteration time 160.05 seconds +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 540 : data loading time 0.00 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 540 : avg forward pass time 71.46 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 540 : backward pass time 8.38 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 540 : optimizer step time 0.39 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 540-------------------- +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6064 iter count: 1.0 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6064 iter count: 1.0 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6064 iter count: 1.0 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6064iter count: 1.0 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:57:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639002 41.718742 41.584644 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.133789 68.371094 67.421875 +util 95.500000 100.000000 91.000000 +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 541 : avg iteration time 49.48 seconds +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 541 : data loading time 0.00 +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 541 : avg forward pass time 13.81 +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 541 : backward pass time 10.14 +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 541 : optimizer step time 1.58 +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 541-------------------- +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:58:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638999 41.718746 41.584633 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.125488 68.373047 67.382812 +util 96.000000 99.000000 90.000000 +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 542 : avg iteration time 45.86 seconds +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 542 : data loading time 0.00 +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 542 : avg forward pass time 11.99 +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 542 : backward pass time 10.15 +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 542 : optimizer step time 1.59 +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 542-------------------- +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:59:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639006 41.718739 41.584644 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.140137 68.392578 67.382812 +util 96.250000 100.000000 90.000000 +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 543 : avg iteration time 45.81 seconds +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 543 : data loading time 0.00 +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 543 : avg forward pass time 11.99 +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 543 : backward pass time 10.13 +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 543 : optimizer step time 1.56 +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 543-------------------- +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 06:59:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639078 41.719021 41.584675 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.140137 68.373047 67.441406 +util 94.500000 100.000000 90.000000 +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 544 : avg iteration time 48.48 seconds +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 544 : data loading time 0.00 +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 544 : avg forward pass time 13.30 +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 544 : backward pass time 10.15 +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 544 : optimizer step time 1.58 +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 544-------------------- +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:00:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639004 41.718735 41.584663 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.726562 107.544922 +util 64.250000 69.000000 56.000000 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 545 : avg iteration time 160.12 seconds +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 545 : data loading time 0.00 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 545 : avg forward pass time 71.47 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 545 : backward pass time 8.41 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 545 : optimizer step time 0.40 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 545-------------------- +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6367 iter count: 1.0 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6367 iter count: 1.0 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6367 iter count: 1.0 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6367iter count: 1.0 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:03:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639006 41.718723 41.584671 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.421875 67.296875 +util 95.750000 100.000000 92.000000 +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 546 : avg iteration time 46.95 seconds +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 546 : data loading time 0.00 +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 546 : avg forward pass time 12.53 +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 546 : backward pass time 10.15 +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 546 : optimizer step time 1.58 +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 546-------------------- +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:04:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639013 41.718723 41.584694 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.289062 67.316406 67.207031 +util 95.750000 99.000000 89.000000 +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 547 : avg iteration time 43.26 seconds +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 547 : data loading time 0.00 +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 547 : avg forward pass time 10.70 +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 547 : backward pass time 10.16 +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 547 : optimizer step time 1.57 +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 547-------------------- +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639013 41.718723 41.584671 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.355469 67.246094 +util 93.750000 100.000000 89.000000 +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 548 : avg iteration time 48.48 seconds +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 548 : data loading time 0.00 +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 548 : avg forward pass time 13.32 +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 548 : backward pass time 10.14 +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 548 : optimizer step time 1.57 +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 548-------------------- +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:05:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639192 41.719444 41.584660 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.313477 67.335938 67.246094 +util 92.500000 98.000000 88.000000 +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 549 : avg iteration time 48.63 seconds +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 549 : data loading time 0.00 +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 549 : avg forward pass time 13.38 +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 549 : backward pass time 10.14 +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 549 : optimizer step time 1.58 +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 549-------------------- +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:06:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639042 41.718727 41.584705 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.707031 107.603516 +util 66.000000 74.000000 61.000000 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 550 : avg iteration time 154.81 seconds +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 550 : data loading time 0.00 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 550 : avg forward pass time 68.82 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 550 : backward pass time 8.40 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 550 : optimizer step time 0.40 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 550-------------------- +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5288 iter count: 1.0 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5288 iter count: 1.0 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5288 iter count: 1.0 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5288iter count: 1.0 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:09:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639041 41.718750 41.584709 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.845703 67.968750 67.804688 +util 93.500000 98.000000 86.000000 +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 551 : avg iteration time 44.25 seconds +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 551 : data loading time 0.00 +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 551 : avg forward pass time 11.21 +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 551 : backward pass time 10.12 +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 551 : optimizer step time 1.57 +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 551-------------------- +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:09:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639041 41.718727 41.584721 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.158203 68.292969 67.753906 +util 94.750000 100.000000 90.000000 +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 552 : avg iteration time 45.97 seconds +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 552 : data loading time 0.00 +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 552 : avg forward pass time 12.06 +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 552 : backward pass time 10.12 +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 552 : optimizer step time 1.58 +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 552-------------------- +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:10:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639071 41.718735 41.584721 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.177734 68.312500 67.773438 +util 97.250000 100.000000 91.000000 +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 553 : avg iteration time 45.88 seconds +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 553 : data loading time 0.00 +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 553 : avg forward pass time 12.00 +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 553 : backward pass time 10.15 +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 553 : optimizer step time 1.58 +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 553-------------------- +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:11:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639097 41.718739 41.584827 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.084961 68.312500 67.402344 +util 95.000000 99.000000 88.000000 +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 554 : avg iteration time 45.97 seconds +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 554 : data loading time 0.00 +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 554 : avg forward pass time 12.04 +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 554 : backward pass time 10.15 +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 554 : optimizer step time 1.58 +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 554-------------------- +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:12:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639108 41.718739 41.584862 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.609375 107.583984 +util 63.250000 68.000000 59.000000 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 555 : avg iteration time 160.55 seconds +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 555 : data loading time 0.00 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 555 : avg forward pass time 71.70 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 555 : backward pass time 8.38 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 555 : optimizer step time 0.40 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 555-------------------- +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5576 iter count: 1.0 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5576 iter count: 1.0 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5576 iter count: 1.0 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5576iter count: 1.0 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:14:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639118 41.718742 41.584862 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.811523 67.949219 67.765625 +util 95.000000 100.000000 90.000000 +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 556 : avg iteration time 46.96 seconds +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 556 : data loading time 0.00 +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 556 : avg forward pass time 12.53 +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 556 : backward pass time 10.16 +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 556 : optimizer step time 1.59 +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 556-------------------- +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:15:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639184 41.719006 41.584862 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.811523 67.949219 67.765625 +util 97.250000 99.000000 92.000000 +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 557 : avg iteration time 48.54 seconds +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 557 : data loading time 0.00 +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 557 : avg forward pass time 13.33 +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 557 : backward pass time 10.15 +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 557 : optimizer step time 1.58 +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 557-------------------- +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:16:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639193 41.718994 41.584885 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.765625 67.734375 +util 94.750000 100.000000 90.000000 +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 558 : avg iteration time 43.26 seconds +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 558 : data loading time 0.00 +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 558 : avg forward pass time 10.70 +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 558 : backward pass time 10.14 +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 558 : optimizer step time 1.57 +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 558-------------------- +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:17:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639253 41.719246 41.584888 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.765625 67.734375 +util 96.500000 99.000000 91.000000 +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 559 : avg iteration time 45.92 seconds +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 559 : data loading time 0.00 +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 559 : avg forward pass time 12.01 +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 559 : backward pass time 10.15 +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 559 : optimizer step time 1.57 +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 559-------------------- +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:17:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639011 41.718246 41.584900 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.589844 107.583984 +util 64.000000 72.000000 59.000000 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 560 : avg iteration time 157.68 seconds +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 560 : data loading time 0.00 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 560 : avg forward pass time 70.25 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 560 : backward pass time 8.40 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 560 : optimizer step time 0.40 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 560-------------------- +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5420 iter count: 1.0 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5420 iter count: 1.0 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5420 iter count: 1.0 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5420iter count: 1.0 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639026 41.718258 41.584953 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.852051 67.871094 67.845703 +util 95.750000 99.000000 89.000000 +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 561 : avg iteration time 46.91 seconds +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 561 : data loading time 0.00 +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 561 : avg forward pass time 12.52 +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 561 : backward pass time 10.15 +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 561 : optimizer step time 1.58 +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 561-------------------- +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:21:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639153 41.718769 41.584927 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.218262 68.314453 67.929688 +util 93.250000 99.000000 89.000000 +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 562 : avg iteration time 48.51 seconds +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 562 : data loading time 0.00 +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 562 : avg forward pass time 13.35 +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 562 : backward pass time 10.11 +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 562 : optimizer step time 1.57 +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 562-------------------- +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:22:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639289 41.719273 41.584934 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.203613 68.353516 67.753906 +util 93.500000 98.000000 86.000000 +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 563 : avg iteration time 43.26 seconds +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 563 : data loading time 0.00 +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 563 : avg forward pass time 10.70 +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 563 : backward pass time 10.15 +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 563 : optimizer step time 1.56 +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 563-------------------- +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:22:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639414 41.719784 41.584949 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.223145 68.392578 67.714844 +util 93.250000 99.000000 89.000000 +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 564 : avg iteration time 45.95 seconds +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 564 : data loading time 0.00 +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 564 : avg forward pass time 12.06 +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 564 : backward pass time 10.13 +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 564 : optimizer step time 1.56 +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 564-------------------- +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:23:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639028 41.718216 41.584949 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.244629 108.423828 107.707031 +util 66.000000 70.000000 58.000000 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 565 : avg iteration time 157.70 seconds +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 565 : data loading time 0.00 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 565 : avg forward pass time 70.29 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 565 : backward pass time 8.37 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 565 : optimizer step time 0.40 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 565-------------------- +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5020 iter count: 1.0 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5020 iter count: 1.0 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5020 iter count: 1.0 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5020iter count: 1.0 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:26:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639037 41.718220 41.584961 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.998535 68.001953 67.988281 +util 92.750000 98.000000 88.000000 +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 566 : avg iteration time 46.89 seconds +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 566 : data loading time 0.00 +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 566 : avg forward pass time 12.52 +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 566 : backward pass time 10.14 +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 566 : optimizer step time 1.56 +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 566-------------------- +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:26:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639100 41.718472 41.584961 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.852051 68.001953 67.402344 +util 96.250000 100.000000 92.000000 +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 567 : avg iteration time 43.25 seconds +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 567 : data loading time 0.00 +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 567 : avg forward pass time 10.71 +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 567 : backward pass time 10.14 +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 567 : optimizer step time 1.56 +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 567-------------------- +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:27:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639407 41.719723 41.584961 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.852051 68.001953 67.402344 +util 94.250000 100.000000 90.000000 +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 568 : avg iteration time 45.93 seconds +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 568 : data loading time 0.00 +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 568 : avg forward pass time 12.04 +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 568 : backward pass time 10.14 +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 568 : optimizer step time 1.56 +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 568-------------------- +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:28:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639477 41.719975 41.584984 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.852051 68.001953 67.402344 +util 96.750000 99.000000 93.000000 +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 569 : avg iteration time 45.88 seconds +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 569 : data loading time 0.00 +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 569 : avg forward pass time 12.01 +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 569 : backward pass time 10.14 +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 569 : optimizer step time 1.57 +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 569-------------------- +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:29:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639406 41.719734 41.584965 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.687500 107.642578 +util 65.750000 70.000000 57.000000 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 570 : avg iteration time 160.05 seconds +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 570 : data loading time 0.00 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 570 : avg forward pass time 71.44 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 570 : backward pass time 8.40 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 570 : optimizer step time 0.40 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 570-------------------- +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6055 iter count: 1.0 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6055 iter count: 1.0 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6055 iter count: 1.0 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6055iter count: 1.0 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:31:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639291 41.719208 41.584965 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.808105 67.826172 67.753906 +util 92.500000 98.000000 88.000000 +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 571 : avg iteration time 49.49 seconds +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 571 : data loading time 0.00 +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 571 : avg forward pass time 13.82 +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 571 : backward pass time 10.12 +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 571 : optimizer step time 1.58 +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 571-------------------- +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:32:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639278 41.719193 41.584965 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.861816 67.884766 67.792969 +util 96.750000 100.000000 93.000000 +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 572 : avg iteration time 43.26 seconds +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 572 : data loading time 0.00 +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 572 : avg forward pass time 10.69 +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 572 : backward pass time 10.14 +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 572 : optimizer step time 1.59 +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 572-------------------- +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:33:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639298 41.719227 41.584999 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.764160 67.845703 67.519531 +util 93.750000 100.000000 89.000000 +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 573 : avg iteration time 45.91 seconds +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 573 : data loading time 0.00 +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 573 : avg forward pass time 12.03 +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 573 : backward pass time 10.14 +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 573 : optimizer step time 1.57 +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 573-------------------- +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:34:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639313 41.719204 41.584988 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.845703 67.246094 +util 93.750000 98.000000 86.000000 +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 574 : avg iteration time 43.30 seconds +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 574 : data loading time 0.00 +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 574 : avg forward pass time 10.72 +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 574 : backward pass time 10.14 +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 574 : optimizer step time 1.57 +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 574-------------------- +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:34:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639322 41.719234 41.585030 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.541504 107.589844 107.525391 +util 65.500000 70.000000 61.000000 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 575 : avg iteration time 162.79 seconds +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 575 : data loading time 0.00 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 575 : avg forward pass time 72.81 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 575 : backward pass time 8.40 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 575 : optimizer step time 0.40 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 575-------------------- +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5562 iter count: 1.0 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5562 iter count: 1.0 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5562 iter count: 1.0 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5562iter count: 1.0 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:37:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639326 41.719212 41.585049 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.796875 67.804688 67.773438 +util 95.250000 100.000000 88.000000 +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 576 : avg iteration time 49.48 seconds +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 576 : data loading time 0.00 +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 576 : avg forward pass time 13.81 +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 576 : backward pass time 10.13 +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 576 : optimizer step time 1.58 +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 576-------------------- +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:38:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639311 41.719234 41.585018 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.226562 68.488281 67.441406 +util 92.750000 98.000000 88.000000 +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 577 : avg iteration time 43.26 seconds +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 577 : data loading time 0.00 +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 577 : avg forward pass time 10.69 +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 577 : backward pass time 10.15 +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 577 : optimizer step time 1.59 +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 577-------------------- +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:39:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639322 41.719234 41.585041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.080078 68.312500 67.382812 +util 95.750000 99.000000 92.000000 +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 578 : avg iteration time 45.84 seconds +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 578 : data loading time 0.00 +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 578 : avg forward pass time 12.00 +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 578 : backward pass time 10.13 +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 578 : optimizer step time 1.57 +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 578-------------------- +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:39:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639380 41.719463 41.585052 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.070312 68.292969 67.402344 +util 95.250000 99.000000 88.000000 +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 579 : avg iteration time 46.04 seconds +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 579 : data loading time 0.00 +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 579 : avg forward pass time 12.10 +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 579 : backward pass time 10.13 +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 579 : optimizer step time 1.57 +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 579-------------------- +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:40:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639286 41.719028 41.585041 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.628906 107.525391 +util 68.750000 73.000000 60.000000 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 580 : avg iteration time 160.06 seconds +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 580 : data loading time 0.00 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 580 : avg forward pass time 71.45 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 580 : backward pass time 8.42 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 580 : optimizer step time 0.40 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 580-------------------- +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5967 iter count: 1.0 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5967 iter count: 1.0 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5967 iter count: 1.0 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5967iter count: 1.0 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:43:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639271 41.719051 41.585018 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 67.910156 67.355469 +util 94.000000 100.000000 90.000000 +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 581 : avg iteration time 46.83 seconds +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 581 : data loading time 0.00 +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 581 : avg forward pass time 12.50 +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 581 : backward pass time 10.14 +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 581 : optimizer step time 1.57 +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 581-------------------- +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639283 41.719055 41.585041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.821289 67.843750 67.753906 +util 95.500000 100.000000 91.000000 +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 582 : avg iteration time 43.26 seconds +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 582 : data loading time 0.00 +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 582 : avg forward pass time 10.69 +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 582 : backward pass time 10.13 +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 582 : optimizer step time 1.58 +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 582-------------------- +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:44:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639277 41.719040 41.585052 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.684570 67.843750 67.207031 +util 93.500000 100.000000 89.000000 +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 583 : avg iteration time 45.93 seconds +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 583 : data loading time 0.00 +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 583 : avg forward pass time 12.04 +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 583 : backward pass time 10.14 +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 583 : optimizer step time 1.57 +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 583-------------------- +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639349 41.719288 41.585060 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.811523 67.843750 67.714844 +util 96.750000 100.000000 90.000000 +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 584 : avg iteration time 45.94 seconds +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 584 : data loading time 0.00 +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 584 : avg forward pass time 12.04 +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 584 : backward pass time 10.13 +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 584 : optimizer step time 1.57 +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 584-------------------- +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:46:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639123 41.718426 41.585041 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.648438 107.603516 +util 66.250000 71.000000 58.000000 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 585 : avg iteration time 163.00 seconds +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 585 : data loading time 0.00 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 585 : avg forward pass time 72.92 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 585 : backward pass time 8.39 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 585 : optimizer step time 0.40 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 585-------------------- +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6016 iter count: 1.0 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6016 iter count: 1.0 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6016 iter count: 1.0 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6016iter count: 1.0 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:49:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639127 41.718449 41.585022 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.445312 67.773438 67.335938 +util 93.750000 99.000000 89.000000 +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 586 : avg iteration time 44.46 seconds +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 586 : data loading time 0.00 +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 586 : avg forward pass time 11.30 +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 586 : backward pass time 10.15 +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 586 : optimizer step time 1.57 +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 586-------------------- +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:49:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639246 41.718914 41.585022 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.753906 67.335938 +util 95.000000 99.000000 87.000000 +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 587 : avg iteration time 43.31 seconds +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 587 : data loading time 0.00 +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 587 : avg forward pass time 10.72 +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 587 : backward pass time 10.14 +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 587 : optimizer step time 1.57 +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 587-------------------- +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:50:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639250 41.718914 41.585022 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.753906 67.355469 +util 95.250000 99.000000 90.000000 +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 588 : avg iteration time 45.86 seconds +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 588 : data loading time 0.00 +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 588 : avg forward pass time 11.99 +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 588 : backward pass time 10.16 +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 588 : optimizer step time 1.57 +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 588-------------------- +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:51:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639263 41.718914 41.585052 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.557617 67.753906 67.492188 +util 93.500000 99.000000 89.000000 +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 589 : avg iteration time 45.87 seconds +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 589 : data loading time 0.00 +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 589 : avg forward pass time 12.00 +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 589 : backward pass time 10.15 +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 589 : optimizer step time 1.57 +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 589-------------------- +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:52:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639256 41.718887 41.585037 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.623047 107.609375 +util 66.250000 74.000000 58.000000 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 590 : avg iteration time 157.55 seconds +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 590 : data loading time 0.00 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 590 : avg forward pass time 70.19 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 590 : backward pass time 8.39 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 590 : optimizer step time 0.40 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 590-------------------- +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5781 iter count: 1.0 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5781 iter count: 1.0 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5781 iter count: 1.0 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5781iter count: 1.0 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:54:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639271 41.718922 41.585068 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.432129 67.773438 67.318359 +util 92.250000 97.000000 89.000000 +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 591 : avg iteration time 46.83 seconds +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 591 : data loading time 0.00 +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 591 : avg forward pass time 12.48 +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 591 : backward pass time 10.16 +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 591 : optimizer step time 1.57 +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 591-------------------- +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:55:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639387 41.719402 41.585045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.705566 67.865234 67.226562 +util 97.000000 100.000000 91.000000 +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 592 : avg iteration time 48.53 seconds +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 592 : data loading time 0.00 +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 592 : avg forward pass time 13.33 +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 592 : backward pass time 10.15 +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 592 : optimizer step time 1.56 +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 592-------------------- +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:56:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639403 41.719402 41.585068 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.852051 67.865234 67.812500 +util 94.250000 100.000000 90.000000 +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 593 : avg iteration time 45.86 seconds +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 593 : data loading time 0.00 +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 593 : avg forward pass time 11.99 +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 593 : backward pass time 10.14 +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 593 : optimizer step time 1.57 +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 593-------------------- +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:57:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639392 41.719402 41.585045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.764160 67.865234 67.460938 +util 95.250000 99.000000 87.000000 +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 594 : avg iteration time 43.31 seconds +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 594 : data loading time 0.00 +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 594 : avg forward pass time 10.73 +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 594 : backward pass time 10.13 +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 594 : optimizer step time 1.57 +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 594-------------------- +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 07:57:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639176 41.718540 41.585045 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.583984 107.570312 +util 65.000000 74.000000 60.000000 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 595 : avg iteration time 160.11 seconds +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 595 : data loading time 0.00 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 595 : avg forward pass time 71.47 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 595 : backward pass time 8.38 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 595 : optimizer step time 0.40 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 595-------------------- +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5811 iter count: 1.0 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5811 iter count: 1.0 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5811 iter count: 1.0 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5811iter count: 1.0 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:00:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639173 41.718517 41.585045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.184082 68.314453 67.792969 +util 97.250000 100.000000 91.000000 +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 596 : avg iteration time 44.33 seconds +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 596 : data loading time 0.00 +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 596 : avg forward pass time 11.24 +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 596 : backward pass time 10.13 +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 596 : optimizer step time 1.57 +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 596-------------------- +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:01:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639241 41.718792 41.585045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.174316 68.314453 67.753906 +util 92.500000 97.000000 90.000000 +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 597 : avg iteration time 43.26 seconds +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 597 : data loading time 0.00 +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 597 : avg forward pass time 10.68 +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 597 : backward pass time 10.15 +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 597 : optimizer step time 1.58 +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 597-------------------- +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:01:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639555 41.720024 41.585045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.184082 68.333984 67.734375 +util 93.000000 98.000000 89.000000 +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 598 : avg iteration time 48.61 seconds +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 598 : data loading time 0.00 +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 598 : avg forward pass time 13.38 +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 598 : backward pass time 10.14 +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 598 : optimizer step time 1.57 +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 598-------------------- +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:02:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639682 41.720512 41.585060 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.174316 68.314453 67.753906 +util 96.250000 100.000000 89.000000 +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 599 : avg iteration time 48.50 seconds +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 599 : data loading time 0.00 +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 599 : avg forward pass time 13.32 +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 599 : backward pass time 10.15 +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 599 : optimizer step time 1.57 +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 599-------------------- +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:03:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:03:32 | INFO | fastgen.trainer:run:225 ] Training complete. +[May 4, 2026 - 08:03:32 | INFO | fastgen.trainer:validate:358 ] Validation iteration 600 +[May 4, 2026 - 08:03:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:03:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:03:37 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.421 +[May 4, 2026 - 08:03:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:03:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:03:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.386 +[May 4, 2026 - 08:03:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:03:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:03:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.881 +[May 4, 2026 - 08:03:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:03:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:03:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.212 +[May 4, 2026 - 08:03:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:03:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:03:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=3.656 +[May 4, 2026 - 08:03:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:04:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:04:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.653 +[May 4, 2026 - 08:04:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:04:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:04:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.489 +[May 4, 2026 - 08:04:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:04:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:04:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.993 +[May 4, 2026 - 08:04:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:04:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:04:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.660 +[May 4, 2026 - 08:04:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 08:04:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 08:04:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.959 +[May 4, 2026 - 08:04:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 600-------------------- +[May 4, 2026 - 08:04:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 08:04:39 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 600 +[May 4, 2026 - 08:04:39 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.131 (n=10) +[May 4, 2026 - 08:04:39 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 600 +[May 4, 2026 - 08:04:39 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints/0000600 +[May 4, 2026 - 08:05:29 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 600 +[May 4, 2026 - 08:05:29 | INFO | fastgen.trainer:run:232 ] Taking a 10 sec nap and exiting training. +[May 4, 2026 - 08:05:39 | INFO | fastgen.utils.distributed:clean_up:88 ] Distributed clean up. +[May 4, 2026 - 08:05:40 | INFO | __main__::49 ] Training finished. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/requirements.txt b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/wandb-metadata.json b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8837aec1ea169d8ab3bc6623d760b14f35d1f219 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/wandb-metadata.json @@ -0,0 +1,70 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-03T11:16:15.596336Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_2step_t833_no_reward.py", + "-", + "trainer.resume=False", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_2step_t833_no_reward", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "b73d18c2e78b2465b81e9263c2223e336a1ea136" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1175529656320" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "x345s6oebbth4ne5yw7ylqsrwuq0vonf" +} \ No newline at end of file diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/wandb-summary.json b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fca65e5494d76eb159bca53f1c15978372395438 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/files/wandb-summary.json @@ -0,0 +1 @@ +{"optimizer/iteration":600,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"val0/sync_c_sample_8":5.660195827484131,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"train/vsd_loss":0.5810546875,"val0/sync_c_sample_2":4.881206512451172,"train_media/student/generation":{"path":"media/videos/train_media/student/generation_500_7abed1a68789457ad3a0.mp4","_type":"video-file","sha256":"7abed1a68789457ad3a077eca4385658ff06eb942ae3ed10f63dbfe4da5df3bb","size":207530},"val0/sync_c_sample_7":4.9925336837768555,"val0/sync_c_sample_5":6.653307914733887,"train/total_loss":0.00879669189453125,"model/total_params":17136849376,"val0/total_loss":0,"profiler/avg_forward_pass_time":13.324691169895232,"profiler/optimizer_step_time":1.5674892757087946,"optimizer/lr_net":2e-06,"_step":600,"train/gan_loss_disc":0,"fake_score/trainable_params":1421383840,"profiler/backward_pass_time":10.146683255210519,"val0/sync_c_sample_9":2.9587841033935547,"val0/sync_c_sample_4":3.6562671661376953,"optimizer/grad_norm (model_key net)":0,"profiler/data_loading_time":0.0009229891002178192,"val0/generated":{"captions":["sync_c=6.421","sync_c=5.386","sync_c=4.881","sync_c=4.212","sync_c=3.656","sync_c=6.653","sync_c=6.489","sync_c=4.993","sync_c=5.660","sync_c=2.959"],"_type":"videos","count":10,"videos":[{"_type":"video-file","sha256":"d4cae118f95943b9957ac5437416f2256fdaab52e79240c7284b4a8ac09c34ee","size":207927,"path":"media/videos/val0/generated_600_d4cae118f95943b9957a.mp4","caption":"sync_c=6.421"},{"sha256":"9dd369241b0d624306b6dd1978e702fd50e2b33c4919cb74959048c61b5b6863","size":205906,"path":"media/videos/val0/generated_600_9dd369241b0d624306b6.mp4","caption":"sync_c=5.386","_type":"video-file"},{"caption":"sync_c=4.881","_type":"video-file","sha256":"46424061a7c159bc5d154dbe9d6cab657dc963d2c99a122663b316684dcd0218","size":200472,"path":"media/videos/val0/generated_600_46424061a7c159bc5d15.mp4"},{"caption":"sync_c=4.212","_type":"video-file","sha256":"56348bb82660a9b2298b618f3b887d1cdeae6b1e873f98daef5a062ae053d0d5","size":287661,"path":"media/videos/val0/generated_600_56348bb82660a9b2298b.mp4"},{"caption":"sync_c=3.656","_type":"video-file","sha256":"57748de0546e2eca3715b13813ece5ec42308930a22ef2e1e49846295311713f","size":246160,"path":"media/videos/val0/generated_600_57748de0546e2eca3715.mp4"},{"size":181424,"path":"media/videos/val0/generated_600_16b78933092d6668cdb7.mp4","caption":"sync_c=6.653","_type":"video-file","sha256":"16b78933092d6668cdb7741ac37d4e4db82dcb646d991bbbe0f99057c759f21b"},{"caption":"sync_c=6.489","_type":"video-file","sha256":"5fad37a16041bbf6fa858dd43a81f2558f6e96f5e742fae3cfe6a11b204e182b","size":286681,"path":"media/videos/val0/generated_600_5fad37a16041bbf6fa85.mp4"},{"_type":"video-file","sha256":"dfdfdd14726cd8f9b0e4c003a910610f887000e6b592766c7e40d94a3d1b99e5","size":190962,"path":"media/videos/val0/generated_600_dfdfdd14726cd8f9b0e4.mp4","caption":"sync_c=4.993"},{"_type":"video-file","sha256":"e3853e4d45d36ffc859e3792969cfe5fb123d1fa52d8c8f88d243e216ed62dbb","size":233977,"path":"media/videos/val0/generated_600_e3853e4d45d36ffc859e.mp4","caption":"sync_c=5.660"},{"caption":"sync_c=2.959","_type":"video-file","sha256":"8cadb1fdd841ccef69841f1f7c5e212b76bd51fd9e3407aee2f3de7620a95f07","size":238718,"path":"media/videos/val0/generated_600_8cadb1fdd841ccef6984.mp4"}]},"val0/reconstructed":{"count":10,"videos":[{"path":"media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4","_type":"video-file","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028},{"_type":"video-file","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214","size":170922,"path":"media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4"},{"_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8","size":173236,"path":"media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4"},{"path":"media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4","_type":"video-file","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372},{"_type":"video-file","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004,"path":"media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4"},{"size":164630,"path":"media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4","_type":"video-file","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395"},{"_type":"video-file","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986,"path":"media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4"},{"_type":"video-file","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516,"path":"media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4"},{"sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001,"path":"media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4","_type":"video-file"},{"size":227490,"path":"media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4","_type":"video-file","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273"}],"captions":false,"_type":"videos"},"optimizer/lr_fake_score":2e-06,"model/trainable_params":2842767680,"train/vsd_loss_unweighted":0.5810546875,"model/local_trainable_params":[711111248,710552144,710552144,710552144],"train/vsd_loss_weighted":0.5810546875,"profiler/avg_iteration_time":48.50046396255493,"model/local_total_params":[4285560952,4283762808,4283762808,4283762808],"_timestamp":1.7778494794217367e+09,"_wandb":{"runtime":42563},"train_media/data/real":{"size":190461,"path":"media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4","_type":"video-file","sha256":"2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae"},"val0/sync_c_sample_6":6.48858118057251,"train/gan_loss_gen":0,"val0/sync_c_sample_0":6.420582294464111,"train/fake_score_loss":0.00879669189453125,"optimizer/non_finite_grads_count (model_key net)":0,"net/trainable_params":1421383840,"val0/sync_c_sample_3":4.21163272857666,"net/local_total_params":[355555624,355276072,355276072,355276072],"val0/sync_c_mean":5.130920267105102,"net/total_params":1421383840,"fake_score/total_params":1421383840,"val0/sync_c_sample_1":5.386111259460449,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"_runtime":42563.172100859} \ No newline at end of file diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug-core.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6d1d046ac131e4ba3e40f65d290016ed222ef18c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-03T20:16:15.721099573+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpup4a1glw/port-2319385.txt","pid":2319385,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-03T20:16:15.721610949+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2319385} +{"time":"2026-05-03T20:16:15.721599303+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2319385-2351394-521295809/socket","Net":"unix"}} +{"time":"2026-05-03T20:16:15.8991085+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-03T20:16:15.914439528+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"pdowrk23","id":"1(@)"} +{"time":"2026-05-03T20:16:16.377418981+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"pdowrk23","id":"1(@)"} +{"time":"2026-05-03T20:16:22.358204612+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"tfqfnk82r0xs"} +{"time":"2026-05-04T08:05:40.278607924+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-04T08:05:40.278792139+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-04T08:05:40.278781405+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-04T08:05:40.278860482+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-04T08:05:40.278884842+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2319385-2351394-521295809/socket","Net":"unix"}} +{"time":"2026-05-04T08:05:42.10511715+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-04T08:05:42.105141021+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-04T08:05:42.105149117+09:00","level":"INFO","msg":"server is closed"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug-internal.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3663c8754fb65fe7b4f46a742480ac18d70d6db9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-03T20:16:15.915170849+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-03T20:16:16.376091891+09:00","level":"INFO","msg":"stream: created new stream","id":"pdowrk23"} +{"time":"2026-05-03T20:16:16.376151425+09:00","level":"INFO","msg":"handler: started","stream_id":"pdowrk23"} +{"time":"2026-05-03T20:16:16.377409672+09:00","level":"INFO","msg":"stream: started","id":"pdowrk23"} +{"time":"2026-05-03T20:16:16.377430695+09:00","level":"INFO","msg":"writer: started","stream_id":"pdowrk23"} +{"time":"2026-05-03T20:16:16.377434325+09:00","level":"INFO","msg":"sender: started","stream_id":"pdowrk23"} +{"time":"2026-05-04T08:05:40.278779245+09:00","level":"INFO","msg":"stream: closing","id":"pdowrk23"} +{"time":"2026-05-04T08:05:41.842704517+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-04T08:05:42.103414303+09:00","level":"INFO","msg":"handler: closed","stream_id":"pdowrk23"} +{"time":"2026-05-04T08:05:42.103940989+09:00","level":"INFO","msg":"sender: closed","stream_id":"pdowrk23"} +{"time":"2026-05-04T08:05:42.10395477+09:00","level":"INFO","msg":"stream: closed","id":"pdowrk23"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..86e4b7084559e2468af7896381713036a114c767 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-03 20:16:15,613 INFO MainThread:2319385 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-03 20:16:15,613 INFO MainThread:2319385 [wandb_setup.py:_flush():81] Configure stats pid to 2319385 +2026-05-03 20:16:15,614 INFO MainThread:2319385 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-03 20:16:15,614 INFO MainThread:2319385 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug.log +2026-05-03 20:16:15,614 INFO MainThread:2319385 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/logs/debug-internal.log +2026-05-03 20:16:15,615 INFO MainThread:2319385 [wandb_init.py:init():844] calling init triggers +2026-05-03 20:16:15,615 INFO MainThread:2319385 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_2step_t833_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '600', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.833', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-03 20:16:15,615 INFO MainThread:2319385 [wandb_init.py:init():892] starting backend +2026-05-03 20:16:15,899 INFO MainThread:2319385 [wandb_init.py:init():895] sending inform_init request +2026-05-03 20:16:15,912 INFO MainThread:2319385 [wandb_init.py:init():903] backend started and connected +2026-05-03 20:16:15,916 INFO MainThread:2319385 [wandb_init.py:init():973] updated telemetry +2026-05-03 20:16:15,935 INFO MainThread:2319385 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-03 20:16:17,104 INFO MainThread:2319385 [wandb_init.py:init():1042] starting run threads in backend +2026-05-03 20:16:17,351 INFO MainThread:2319385 [wandb_run.py:_console_start():2524] atexit reg +2026-05-03 20:16:17,351 INFO MainThread:2319385 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-03 20:16:17,351 INFO MainThread:2319385 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-03 20:16:17,351 INFO MainThread:2319385 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-03 20:16:17,357 INFO MainThread:2319385 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-04 08:05:40,278 INFO wandb-AsyncioManager-main:2319385 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-04 08:05:40,279 INFO wandb-AsyncioManager-main:2319385 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/run-pdowrk23.wandb b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/run-pdowrk23.wandb new file mode 100644 index 0000000000000000000000000000000000000000..969f5b280d591ec89b9f063093bf99b76f4eeb3b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb/run-20260503_201615-pdowrk23/run-pdowrk23.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b25025432ed8a0c09f9bca0b633851bf80d48015e7e418770d93d90cb3221110 +size 9396153 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb_id.txt b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..6fa4ee24c324c72dbc7f83777ef7527adb2dc22e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_2STEP_T833_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t833_no_reward/wandb_id.txt @@ -0,0 +1 @@ +pdowrk23