diff --git a/.gitattributes b/.gitattributes index 500964c76ef5f54e794f65d1bc64ac441a47d103..116cd2b45c9b4c448b7455645231fdcd5d60ef11 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1259,3 +1259,235 @@ hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_ hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/run-lvwg9pfc.wandb filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_100_311ee73b1decaeffef68.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_200_d06337493e6bb8794e7c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_300_bf8c8920c2182caaf9be.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_400_3e7c7aa768c41cd1ff4d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_500_54d7e0b7b8631130c013.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_38e1707fe961b33c5b99.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_46a86ff2381e3f273c48.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_571057fc70e90a629d61.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_91999b86864c0f722f9f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_a5421f80f1fa773b5308.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_ab7387fd03af17b94a61.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_d5ac67b4bacf4633ca4c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_ead0188b8180b63b1d28.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_eee30a244a30b19f865e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_fd53748c7bfaefc78c05.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_247e20c38838e1d1a5b5.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_4112de6cb69227473a83.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_59e4d23b6153e74da913.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_63503cf577196e3fa498.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_6977326027417888efea.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_a480f7b179e72895ce09.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_b304784f140c3d0078d6.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_b5483a982b1859329da1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_e72bff9903269cffb9b1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_ec23213bd7149638dd25.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_140a166b247402ba651b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_1db3224e86035db4e7ce.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_2656b9553ea41c2dcc9c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_336f36825bec3f9ee6ae.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_340dddb7fac6300d17d8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_37686cb0d4b8d0324491.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_3b4b4e57b116aee7c779.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_4fd0096272ebdf695289.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_b551980069fe4e52cc5a.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_ba3e57695bcf53a7a979.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_1934a7ced88c6241f044.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_44c371250829a86229f8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_95028884d982fb8a0cb6.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_959c465e3f3e4bf8f583.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_9e7d01279fb0b2f29fa8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_9ea99436f5871458300f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_b39f29a802d827e4c574.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_cb6385243703cab00b9c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_cbda3b15d713982cae33.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_d4710ab9f8ec6864adf0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_091b5abe174ec3eec207.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_270ca0f6fafd21499ef2.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_391359b72471172718e7.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_746f1da4cd90b7daec19.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_84d3bccbc62388a00374.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_95e3a497e5ddd6767e31.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_ad311278e6b48ff2f590.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_c066d67bb8d747571584.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_c564f693bab8391cec26.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_f433153364fd254cfe02.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_13dd880e1be3afbc4305.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_1daf3ffd78eaedf00b93.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_30ebc58e2c2585a478e2.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_5233db7fa378d994b2bf.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_6213360a5c574ef5b5da.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_66e24e5fe2e1c0fe4f8a.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_86754b3d2902e460dd2c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_c634261dd6b7f89f930e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_c7bfada6d2a58a176ceb.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_dc0f8b7980bb0406ab08.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/run-lw2bm1l6.wandb filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_0a9e7fd50394d8317c8b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_195f968c26d2d073200e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_279df03ae1fa72af4482.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_3b5f4dc6836b99443638.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_742edf4db049b9a7a78c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_8dad6b26232c7833b2a6.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_9f9787ff0ab4e25b8d0d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_c77651e7527fa1dcdf2c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_d5a22a2ad522cb8f20aa.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_de88448d1dfb6875a6d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/run-lw2bm1l6.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c7f7a142ca90adf44920ff529aa2e22f02879f64 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:530a65dfbd8b2b2e565a5b9ad0308761da47acb18fcf6dc4475577507f217fa6 +size 635541 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fec3e0ddc631c3b5725d51fcb101b412be117260 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5def828768f64e70dcf794b9ab2b5f9c6f78a04dc311f8d1eaa48ecbbc027fdf +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0444978ba0476b63ec6cbddf9ccea427850552a4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4779091de5637b120fdcf461efa367086552551d4db42d8365ed05a6a7ea973f +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..90801cd3fc95b2e03359a725b4161cbdb6b69f8a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:050bd13cfe6d01b4df124fdaf76e55102ac1ec8a2c3fd9789ea76a1151339ae3 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..24aa1756b0f6c6e2e6c6611b04850af7b47ed17e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d8d50cd17ec464d69d40b5e8e305aa3bfe9b5188c094cf6226400e2aa37e3d6 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..0c256ac9bfd012f5cea5007e62ac62ad0ede69de --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263da24d3ae3aad2cfc5b8707aa509b8e33b7173749749a8a34f20be5ecdec6f +size 635534 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6f2fdaac49296374e92368eb9d3ae4916d656b29 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:507e9723a71bf8f6d760b0bc4dfa57614baa63cd59fb36d40a89ab72ed87f055 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6e76371def01162bf18534e56ed9e1de73a14db8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d1782d5fb1852077b0dd0763628c4521600379b785c5a34d65f9aad2651b1d8 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..371137cccd5a65144b016fd0043bfcbc5131f9c6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80afd48145054ca650878ffd27a5b66ccafe0afba02e92b2466251d6f3de3c8a +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4327c9b7be20a1faa9f250970003eabde638533b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8d47dc95883997d0022f9e178334a18ed1dff7a8729e33ff979892f7b3cf59c +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f637e0f6792d8ae1960e0e448573321eacbddc4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9e7bf7b483afea9b5f5fc9f6d1a8368eea2246e5e5404c08b9fba8acdc0064 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6d94c7675be0c07e4aaa012c1bd183577acfef62 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a334075eb1ba17c58f8da768fdbd3f3a23ea0a1d6c45e6ff77cbe621b6e44b7 +size 635541 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f2923de81e30573b5424abd4e31269687c4d3c05 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0b88a349615d21a2beee67fc1577ebfc4a171d26798922ae65149fdd6c1c72f +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d5e47a535f5e77b5626421b9a1b31d1a43717943 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d943f3deff603d1fb1a0bdf70514a7584ebf1e489002a104cbd3610e93f7290f +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..437d2875f544e3be6444fefc9a5fda1b62177ffb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cb9aa9839285c9756f48b8bcd257196c2e84d820c60e8b54c93c2bce86f9c9b +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3362d30ae44d04230c9c7a7df2161c4bc1719ecb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20e85b7d85195dc5a232b324850c2ede8362165c1a3f5b597625c30092627ede +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..e0107ec8f854c8134312bd2396bd42adedf91c0e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d271668a0a0565da10d17b2d304a79cf15ee6c2a3d4671edc72fd329c3f3c5d0 +size 635534 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..cd0bc52958cbf38d4c684a1ca214b01e3b3a8e9c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43c7aac756d67c21bad821b0e63b8c04f038d5b862d9144a0a7dbcbcaf6637dd +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b0abe6bf4bff4f6d03ee826c9f587d79911b003d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df766f41ade61b2347b476b11e0e694a65393decb1286479d629eba8d789ef4a +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8500977ce5eb6dd25ab441330a0c25a32a09bf22 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:940190cefb1ed6b765bb45ad6463516bfbe2752b2cdf2ff48246eefb2378f8e1 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..03e0d2a8ab2493e00beebaa49ae1a77cca60e1d9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7629c275e9478e9414953b4da79af486ac7e7bdf478e3be31449318bb4e4b426 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.pth new file mode 100644 index 0000000000000000000000000000000000000000..94689c6d88777305ec14be3e82dcf676c28c81a1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d49c29dddf4cff9abe84c331dfad17799824bf4158c145f77812c5430228f1 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d800e13ac0fa42cb850465a109a2ff0faed0a0e0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:977bba4279133411aa7b08d9a7bb7ca4d7e9a2feed47d73c32c892f8c25d1d9d +size 635541 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..aa2ff52f7ab015c0848be7165cfee1601c6b3c83 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21c358b376d3a2f540006b4d9b8625b54773b61dca8499c2f81db0682933bcfe +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..5d6a32db4575362f8c5c940084c8d796faa3da1a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f212c8e7eeb23ad5997eba1971977905b8bff80d42f5898350f56bc327361b1 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8aa62085df250b1a55d971147efb67fa86f2a299 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54ffab84ce15f6b5ecb4e6c65287b4d083c3738e85c490e151262a528d8343c3 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..50b99ae60db02836db83f6a9f26e575953079d2a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65988a45e0c14f33893740d144f8f720c4661cf488e01605c0763b525d4dd313 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6d4aec92dbbc51e431799ca982e326be9eea5bc8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:806c6d699be9d9226d1247c8e38cfd1eec6e30a1991015786629402390af6c38 +size 635534 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..038a85a3979a278179265291a440ee739ac6c0a9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:911ea636be05c85018844aebce9d7154c57beb6afa52161fc00a00a32a82001a +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b0fa4c0ba27ee682b3c75eefb6385dd47d08d587 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdcf4b21042d7c99020c1c150f0cfe1b564c5f9ce17711fb5ed09adf0e0916b5 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3d3f6cf5505ddf15c6278d1af8ba7f0a0c4e648f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e596833950a58c8103c8a2d99d26c269629a04e52e5db88088e1d8609532dffd +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2d0e4b9b8fae14ec1cce233f32fe5b07b363de74 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:244fc6ae1677c9b78f95c798cd54a225214bd19615495cbd85cfe739c001189b +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.pth new file mode 100644 index 0000000000000000000000000000000000000000..36cdfef63708c11e993e5164fd9859eb64d6d33e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463104a57d6bfde142e5cd01a059afdb8b99dad40c73e04a236502ee34fb4d66 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..3690d1dd0c4eaa708cbb194a4a2ba8c160c531e8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:642150ca28e9b805a6cc92a5170d457a21e800b9f96dca4cb5ace4d68e804dd9 +size 635541 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f3db66bd9b1b629861bd69651eaaceba0c76624a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:790cb84c371bf82e01463a565990d448ec793533e5f2feeeee59e0b498d7707b +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2e91b8343c6de94e45e0a9e6d5543d533ab4993f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb6eb02adfc9b17b12c01d4f2edf8ec141d1d1e6713b020d641bf5f7b832e414 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..92c2574f5cfedb03fecb9f4ce0868c8c7c0c0840 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f0a13f20d5cf1fda387e6144ffc5609dee3fe04aca20e9717413933161e76c5 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..55b71da7b94dfc328594558c56a0c6df1e585d82 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d26b5132f1a415eea15fec1c246425c3204dc5ee08d35e0888762131511b8998 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c3e957e080692f1b4331aed24d26b67fa79ce500 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a64fb74446af7b475b3c211e675febe71a102d118e79425cab25183555fcef0 +size 635534 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..929212c4800d65e5f154fa2e4df46fed5a309934 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76f24f649d97136976d822f9196597e0bdb9e3f4ccd19ef32b76022df81be189 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2ad3716da52f24bbb3cdd281ed9516aeec7f51fd --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b11307294f8750d7236d1618a59e4e4cd980a113fa5df570357470181ea79aca +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c24aecae5bb4b2a7dbf88932e00abfe98e69a0b0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28a331fe07d7536daf3103d9460a91bc8d359b217ba4c6ba0ce537dd0681ecda +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..aee23d9a248bab051ca78b3275f2d67633b82181 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b23eba6faf176134744ff517f369c53c08aa89fcddd39054d85c5a5906d71f4 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.pth new file mode 100644 index 0000000000000000000000000000000000000000..0deb1499fd51197d9815f644a93381fbe0678161 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31545bbbd2979dddf082789dbf999d9324dc9febf1b016304ddc713ede429c3e +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..80c7c38f27f13e382d6965bb6abe0b81dd87c99f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b229816f1b9de104cee33315503b150ab761a5d2469755d5de13aee7e78346c +size 635541 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..99663a51928a9fe71361fdf5514a251a4699647a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2e648343494fc9642224288b2d969c25295db38552aa7c5f808b1ab1e39dda3 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..aef5e1391fc4b844bf3a181632f559eab07e92ed --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfeb1f21ccd84668a87c6bd8d5eb55fab8f40bd8f690eb926e4939bf38c9b539 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6d840d4237b9bd6e0b3bf60e5e1ab6d8dba739ec --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a21aefdaaba3ad606bdace79ba221c499f1721933ffb5c574d8d2b5abfd1be06 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8a259399ebe529bd09f0a22270f013eb00314b4f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fc6ff3b5b13f6d5422d2c7e60958d32ca00bf570a51428b23784f76bb65de50 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..04a5148c579a9fee6cb3fcf770ecdc928579ae96 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23bedbb663bb883c4e8cb3137fd5b5d33eb96ba77f60a783a3b7b5dda9c72100 +size 635534 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..056ff91ecc59ce6ce5d2a39dfa55c91e3ce74d03 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce8c427bb5f64eb4e71d491b295bdeca2230bab2269a74cb975c1b4ea45edfbd +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d28cf3ab406041f91a4fa89a3faf39e957f54e77 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0316693489627aee9af67e2d41311ab2db054eaaeb6f98feb9734715868af8b4 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0d158124083348f2db7995cb4fe1a2e88f68536d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f359bb31fed4b3cdb967ee93f3bf63c5a5f59e14b9c2b1f65c96853c8c74313 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bcb69d352a4f4b019167922c03de6412ab485e6d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a5305a6e6f183d60028b377ba4dddda2d937f244438b8f90f62404ab4fb1aae +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..df0fb483027e40ac33d6a36f4b38fcfd244d6a44 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2110994f1f90ec15f3f831251e70d29fe1b83f8a35bb969069c4ac3f211c6d +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..675ce0ba2aff9b083926266387ef36da4c64319f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6727943c3817c361ae7a2274a491031995857afe5b16813db129991ae405650 +size 635541 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..67606138513a7d306051b2dc27f23aca9b944f25 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6634bf9e203b705a445081f795f75ae4e37708f58f3b3c8490dda3cab642f59b +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6a83d735edec18ff0698b926f903160540c2929d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fe691e8e0ff10923a9876142ff51a596b2ecad3d6ef29ffc62df1f8a9f80124 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..dcddb5a4914d60f9ed2ff15ae1a7abcc9ca3cb32 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f918a70497abb282a4473ca2484eece1d7e84dd6f1557fefbfd6583cead70666 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8783848c268ce5ddb707a74f3a7f766e1b07f9ce --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9719e8c6160eb5674cc958b0f38927cc1e7058087b3b9980f8fd095b256bf002 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d4682e6c13779c825cc3037023a9b2be13a95428 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d49174cfa5201de42461136a28bed187afce78f108868b3215500920dde9b69b +size 635534 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..861c9df5248de8445d75a243e61ddfcedeb230ce --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44d62563b4d6544a45efcd08c1155d435a42feeff31afe6269c0fa611a4af79a +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7bdbb316310c9804556a6006284b8f244bce0715 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3b4527c8cebb73e7afe588788019d9e2216e1b94cd64ce05b78c4a08b1b4037 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..792191c187eb493ddd19f421615e5f401f377828 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6c369f305eb59d01d7a25045a7e764d5aefc5de3a8e971f56f17c2e2e9d625 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6adb3cfda892e70af71ae484b6080a4a592c2616 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ded5b0c94138eecd221c70958d745ea253c5b639dd57624f3822bfc14e550b6e +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.pth new file mode 100644 index 0000000000000000000000000000000000000000..97cb41dfed21f440317a176863edebc119d1a324 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c1c25fd6c3ea2819196abcf7cba095dde0d86545b36426cd731e4feee3caa8 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9b12caee7c26f7909e7570111de7d045076a8f0b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e1b45a71b99a9d79398ef95728951fbdbbd83a0d3ed6879e1812a2fc4416472 +size 635541 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9323da170f9f68a20b6cc3e809539fe7a59b6ea4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0336ef86b2855aea94cfbc1cce755f9e0f804c4e61dff108a1df016fef5cdd34 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..5c5575465d14a1852e241a364a4eb64ffdac9e09 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dbf0c1bbf1f5d68c85d92d2e61c260ee71e57a21dd132a04c519388b141f9a8 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2e36d09ab3d4acc11131a9b49819d78c8f6e6b67 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916b2a5271df34e5461288ac4e08e17b7d2e8887e51c1747c4e374e2148f7f53 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..95b5df042b63a14ea0a61243ee77edbf84e44cea --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c02b49a83b410452f04681ea815cde5e2ff86dcd31be993ed8b7f0cbffea7a1f +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c4e1e634f09607df0ebed8f945c77fd15973a5b3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31d51db3f56a302f912072636dda8f7d17c5d37c346fe2be929f4c9155dbcda4 +size 1722905 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f15f47e61aa74245e22d3601527ff497bb48718c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e231f513999d68a0e20bc37507bff92d21c8cab5ee9e957e4e852083fa6723 +size 2847149355 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9a935650d14507c36bc66cb00510e376a1238430 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16c5f469fb0f8896e7b8e129182051390611f299b1f9f1d87232bbb57c45d0b3 +size 2845267132 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a8c6292f5a2b0f23ae7c246b1f60c066c2b93a2b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bdc4dda755056ed6680e8bbe38d8432bc35c29d5e47743b08b571b459ded02e +size 2845352958 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c28800c3898168a7f1d2ca63ea734fb3004db9d7 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c22c1b04e5cb5f1f1ad15dd288101f7f7aee48d9885188da6d78372076e95555 +size 2845316241 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1d23fbf4920c6264d7cc54b5134b03b03add6043 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2846790d16d780c5a3a8ef6217367037c9fd8632f4170b1d8ed946e2ce59f1d9 +size 635534 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4782a9c165411f1e81af7f8202c2df7392c1bfc1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c323cecc51002060b8fe7173d46391685c7b5ef48fc9c6d1f9e1c65424226aec +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..48601019f985328cbcf496ea282c689bd329afd0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3f776f6cfa7b78a82e890baf113262b0e0d154578e38a73aad63e2601a540bd +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..91d8fae4e978425f7527062733211821d7e47809 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3a087bd9d2c6176e160335749488aff328be6454231b4c919edd3361b0b3368 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..244e8d007708ce1ba50b9a9e973818f39bc2fc61 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd2d34229935df2d32634decbdbf35dfd84115dd75a3df87a2912c4d2ea29168 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..af39f975341150f46a3449d81af2f3d200e12b47 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44acee2cfca49ad32dbf926bac03b2c7506750bb5be553a6f082436d879337a3 +size 1722748 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a7ff6e6e7419c4954e877c4bf70117c5c356d2bf --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f92de15d765b20c95a85e13755457394fd6443f8fae158d526d70e26a97c553a +size 2847149355 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..280e976c777bdd753a29c5ce5ec1e650d04b79ef --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24ead14bc101bff52c2eb50a46bac684529d2016b6dc226bbf181d8f289d4a49 +size 2845267132 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3e3555e4c77bf51a628614620f5c4160d272c9a8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5b428341f0a2a3c46f2c92ccfb8c7cd9aaa1eb89437a0665ac3f7740d3ab979 +size 2845352958 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fbb165baa80ca3c236dfe8f9829c97824360dcb9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d414c2de330d4c4601d18ada8c9c843a7fa668fbce439f8a2e9009d6006ef39 +size 2845316241 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth new file mode 100644 index 0000000000000000000000000000000000000000..20e4f38a6d8cc3aa8546b004ca664ad199f33358 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d438cc8922e447919e87494a29301517f583bd61e7ff6182b950306238a2117d +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8efca30d2f7d792803d37491a3a3e388d6cc3985 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'False' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '0' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.769' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '700' + offload_module_in_decoding: 'False' + resume: 'True' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug-internal.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7d9da8d01699b655a105b34d3cbed7c971bc118e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-04T08:10:11.332926177+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-04T08:10:11.770436324+09:00","level":"INFO","msg":"stream: created new stream","id":"lw2bm1l6"} +{"time":"2026-05-04T08:10:11.770484268+09:00","level":"INFO","msg":"handler: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T08:10:11.77160046+09:00","level":"INFO","msg":"stream: started","id":"lw2bm1l6"} +{"time":"2026-05-04T08:10:11.771618514+09:00","level":"INFO","msg":"writer: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T08:10:11.771631004+09:00","level":"INFO","msg":"sender: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T10:06:01.242751452+09:00","level":"INFO","msg":"stream: closing","id":"lw2bm1l6"} +{"time":"2026-05-04T10:06:02.302129532+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-04T10:06:02.591565787+09:00","level":"INFO","msg":"handler: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T10:06:02.592126222+09:00","level":"INFO","msg":"sender: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T10:06:02.592143864+09:00","level":"INFO","msg":"stream: closed","id":"lw2bm1l6"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5ae4c09fed3e52e66bb958ec23731fc9bc1b0157 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug.log @@ -0,0 +1,22 @@ +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_setup.py:_flush():81] Configure stats pid to 3985335 +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug.log +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug-internal.log +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_init.py:init():844] calling init triggers +2026-05-04 08:10:11,035 INFO MainThread:3985335 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '700', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-04 08:10:11,035 INFO MainThread:3985335 [wandb_init.py:init():892] starting backend +2026-05-04 08:10:11,316 INFO MainThread:3985335 [wandb_init.py:init():895] sending inform_init request +2026-05-04 08:10:11,331 INFO MainThread:3985335 [wandb_init.py:init():903] backend started and connected +2026-05-04 08:10:11,335 INFO MainThread:3985335 [wandb_init.py:init():973] updated telemetry +2026-05-04 08:10:11,358 INFO MainThread:3985335 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-04 08:10:12,422 INFO MainThread:3985335 [wandb_init.py:init():1037] run resumed +2026-05-04 08:10:12,425 INFO MainThread:3985335 [wandb_init.py:init():1042] starting run threads in backend +2026-05-04 08:10:12,705 INFO MainThread:3985335 [wandb_run.py:_console_start():2524] atexit reg +2026-05-04 08:10:12,705 INFO MainThread:3985335 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-04 08:10:12,705 INFO MainThread:3985335 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-04 08:10:12,705 INFO MainThread:3985335 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-04 08:10:12,711 INFO MainThread:3985335 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-04 10:06:01,242 INFO wandb-AsyncioManager-main:3985335 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-04 10:06:01,243 INFO wandb-AsyncioManager-main:3985335 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/config.yaml b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a5a15bd766ef73515da65defb0cd05adffa89a9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/config.yaml @@ -0,0 +1,387 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + ybjk1atzxhtxhspjhbgv689r015pk5wo: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769_no_reward.py + - '-' + - trainer.resume=False + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_t769_no_reward + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1159903330304" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: f5897815a4bdcc8f11212b24f367a6839a5409ca + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward + startedAt: "2026-05-01T03:59:36.507877Z" + writerId: ybjk1atzxhtxhspjhbgv689r015pk5wo + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "False" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "0" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "600" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9684c7bde9ed18086d39e7a9880b642c505b3947 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112c0905448a821974f0bf0a94d9107473bb02b60e0ebe1e75a57d309670facb +size 219311 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e38036c10e966c2c82fa717b29431a1ff88e1ae --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae +size 190461 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_100_311ee73b1decaeffef68.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_100_311ee73b1decaeffef68.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3e8831d0630d421d8c0c25a28f7f82cfa2e1f0a0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_100_311ee73b1decaeffef68.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:311ee73b1decaeffef68f42c0076e4d5f943320a1a47e1f855291f60ab20fc0c +size 188535 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_200_d06337493e6bb8794e7c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_200_d06337493e6bb8794e7c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b06d9f0c42a80b57e66a813b027bc882432d2aea --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_200_d06337493e6bb8794e7c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d06337493e6bb8794e7ce6d844f1ef377fd67f7eaa7c2e7b954993a1c2393638 +size 193168 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_300_bf8c8920c2182caaf9be.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_300_bf8c8920c2182caaf9be.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d1358d2ebf230e42df2a89e48e0d5b2d01f77410 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_300_bf8c8920c2182caaf9be.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf8c8920c2182caaf9beb2397818e1b04a332b7d4cc1553b3cf20de0d0f76ce8 +size 207220 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_400_3e7c7aa768c41cd1ff4d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_400_3e7c7aa768c41cd1ff4d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f654108c6ba9289ba39afad8a91f58fb6411b58d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_400_3e7c7aa768c41cd1ff4d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e7c7aa768c41cd1ff4d5de365c41eb108187b0eca172786e86c3b1d75a0c987 +size 233038 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_500_54d7e0b7b8631130c013.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_500_54d7e0b7b8631130c013.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b2dbb56253db0ca5949fc9b4030412097766daba --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/train_media/student/generation_500_54d7e0b7b8631130c013.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54d7e0b7b8631130c0130dc593aca13885eead1f40bb955833b5027567d55eaa +size 215988 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_38e1707fe961b33c5b99.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_38e1707fe961b33c5b99.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61e4b33da92abc4129682301221aae0c4e9e446f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_38e1707fe961b33c5b99.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e1707fe961b33c5b99bb21a07341aa578f6afe520b8e88271c2fc9e770a011 +size 220611 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_46a86ff2381e3f273c48.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_46a86ff2381e3f273c48.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..522cacc12f03185123797c5dec9219c540795844 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_46a86ff2381e3f273c48.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46a86ff2381e3f273c484b107ddebeda852cf15cfcb3adf99a49e1b0c09f7973 +size 248727 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_571057fc70e90a629d61.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_571057fc70e90a629d61.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6f957aa0a41c12aae78523b3c42b13e8e5c6f778 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_571057fc70e90a629d61.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571057fc70e90a629d61e8c2f0ab9b6a23e20ca1df44b5bd00b931e42585ac90 +size 183800 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_91999b86864c0f722f9f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_91999b86864c0f722f9f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cc1f34655a625aa2660a265f0e93292a40b28d68 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_91999b86864c0f722f9f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91999b86864c0f722f9f3e0ab234d41878aa46f66dc9c93ae1481dfcd349e2fa +size 263493 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_a5421f80f1fa773b5308.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_a5421f80f1fa773b5308.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e456ff905b3fd81ba425e30518e0e85e4aac5295 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_a5421f80f1fa773b5308.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5421f80f1fa773b53080fc21f217af2fbfb4a0ba7683da38bbd1846b7ab3bc5 +size 172508 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_ab7387fd03af17b94a61.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_ab7387fd03af17b94a61.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..21451cbbbbc4d58845e8804f2c45bfc85ff2502b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_ab7387fd03af17b94a61.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab7387fd03af17b94a610e319510bbfa3743d8fad2cd7e3b74f645edccb4ea15 +size 213781 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_d5ac67b4bacf4633ca4c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_d5ac67b4bacf4633ca4c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f153e66f7efe2dfcf5154117b6b75f4526c9917 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_d5ac67b4bacf4633ca4c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5ac67b4bacf4633ca4ccb95675c3a346fbb665798f92444a69c0bc607617cf1 +size 190629 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_ead0188b8180b63b1d28.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_ead0188b8180b63b1d28.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..84884b0a7300ad6c18c2b1b218fd02ec3a84d96e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_ead0188b8180b63b1d28.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ead0188b8180b63b1d2882e5d096094a031cabdf5721537e463e9437f97ac263 +size 178286 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_eee30a244a30b19f865e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_eee30a244a30b19f865e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a02c108cdf792c4d5481b6b728dc300f0c7de541 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_eee30a244a30b19f865e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee30a244a30b19f865e212a32feefa63e610ada3e446c1ef59f7d7bd7dee779 +size 215379 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_fd53748c7bfaefc78c05.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_fd53748c7bfaefc78c05.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6be76e542bc054fbe97ebcdfa256a21ef65475e3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_100_fd53748c7bfaefc78c05.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd53748c7bfaefc78c05b98f301b85d2ec49e6143d818c2e0779a7ab83f5c006 +size 161609 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_247e20c38838e1d1a5b5.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_247e20c38838e1d1a5b5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f65233934107ea4af95b739a9db597c51b91f298 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_247e20c38838e1d1a5b5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:247e20c38838e1d1a5b5f8d132896241e6e10cdebcce060b1a88f2f458839414 +size 217944 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_4112de6cb69227473a83.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_4112de6cb69227473a83.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..54eaf6c91d1682c7fa7b96d0e20ae53bf9f0c460 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_4112de6cb69227473a83.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4112de6cb69227473a8351350736cd6beb88b8e7a6d413d776d9b9db1d24ebb9 +size 188602 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_59e4d23b6153e74da913.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_59e4d23b6153e74da913.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d2b665adf273aa6d41321165a22dd1fc5762deb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_59e4d23b6153e74da913.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59e4d23b6153e74da913d9c709873b1c52c25e277a9a4f19d8a7d90f18841ab4 +size 225549 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_63503cf577196e3fa498.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_63503cf577196e3fa498.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8ce96dc75630ec9940a71455e59482da100bf5c1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_63503cf577196e3fa498.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63503cf577196e3fa4989f0f9248579a69dd24b633dc1049bd49dc2658dfe65b +size 177171 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_6977326027417888efea.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_6977326027417888efea.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f63b7f3519e7647379a9a1799136acbdfb1c96e0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_6977326027417888efea.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6977326027417888efeacfc39e86a66e2d392cbd54406837a70ad85581bb5610 +size 255945 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_a480f7b179e72895ce09.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_a480f7b179e72895ce09.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f475435387b5801aebb8548d18c0130afa4b3619 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_a480f7b179e72895ce09.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a480f7b179e72895ce090878928a86ce4f3c373dfdc870e5184b05495087f1a7 +size 166492 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_b304784f140c3d0078d6.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_b304784f140c3d0078d6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..be4b6623400d623ef7d7bf190653aeaa4b1be385 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_b304784f140c3d0078d6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b304784f140c3d0078d602587ce4f25f6bcebb387f4ccd8243d224750b973a76 +size 267672 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_b5483a982b1859329da1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_b5483a982b1859329da1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc4df229f94f0fbbfed1bea10c5a35026b580da2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_b5483a982b1859329da1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5483a982b1859329da1c54fa3d1c4c41b715da07dba25db439d46ea5a8d44d1 +size 224149 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_e72bff9903269cffb9b1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_e72bff9903269cffb9b1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..208cef68659f5062e15a0579fb78d3b2640aa8c2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_e72bff9903269cffb9b1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72bff9903269cffb9b1b9bc87ec583e41265ef4b33de2ff69fc81706fabb302 +size 193869 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_ec23213bd7149638dd25.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_ec23213bd7149638dd25.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c4bd4287b17d4f013643b7526295ac67ca63985 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_200_ec23213bd7149638dd25.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec23213bd7149638dd25f3cfe09fb1bdbef05899e9f027d621dcbc17f1f5b61a +size 182823 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_140a166b247402ba651b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_140a166b247402ba651b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aef7b9889929227bfe53b5696e5a8f3769234318 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_140a166b247402ba651b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:140a166b247402ba651b9bfe153b1559d321df7546432250bd650a950ec1cd36 +size 281670 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_1db3224e86035db4e7ce.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_1db3224e86035db4e7ce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ca918e25149d4dfe623b1fcc4e56c51aa194d0f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_1db3224e86035db4e7ce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1db3224e86035db4e7ceca2f428abaa63db73ba3b0a6c0870825ba2a3ab517cc +size 202053 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_2656b9553ea41c2dcc9c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_2656b9553ea41c2dcc9c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..076e8d398995b91c4ced2f561d33ffa708c99026 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_2656b9553ea41c2dcc9c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2656b9553ea41c2dcc9cfd8d4596a7efdb1f95019424c651577d999f40992a55 +size 172286 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_336f36825bec3f9ee6ae.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_336f36825bec3f9ee6ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1dcb92f9a709e0e515f23a70d9c0260308f49cc9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_336f36825bec3f9ee6ae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:336f36825bec3f9ee6ae7daf308ca6119a8e8563a1da49760b17ac84c167623c +size 230229 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_340dddb7fac6300d17d8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_340dddb7fac6300d17d8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dda3075ae46126488954c19f128e7328d67e9686 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_340dddb7fac6300d17d8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:340dddb7fac6300d17d8bd817bf4fad3d30624079004af30f9b89b845b3ae28b +size 223332 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_37686cb0d4b8d0324491.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_37686cb0d4b8d0324491.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b568607c1a85eeeb9681048546f57867f03eb626 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_37686cb0d4b8d0324491.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37686cb0d4b8d0324491c8e1e5b60c3076cb89b5998691345f07cd73028cfcf6 +size 268861 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_3b4b4e57b116aee7c779.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_3b4b4e57b116aee7c779.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..27a574ceea217c76cfbf2dd4ac644a49b98bdbb7 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_3b4b4e57b116aee7c779.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b4b4e57b116aee7c779f880256440645984a1b0c0fffcfb13c00c91c209e69c +size 234343 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_4fd0096272ebdf695289.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_4fd0096272ebdf695289.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f45c7f3f5ffad07e80dfbc86245c049de8d25e50 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_4fd0096272ebdf695289.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd0096272ebdf695289f89fc338b482115af7dad81cad804d651a102963913d +size 185108 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_b551980069fe4e52cc5a.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_b551980069fe4e52cc5a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f9f317b077e5fde5925cc4906cec6f3534ed297 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_b551980069fe4e52cc5a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b551980069fe4e52cc5ac3bc8f260fed7965eb18ba2653b0c08ec5b0aff59ae8 +size 193052 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_ba3e57695bcf53a7a979.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_ba3e57695bcf53a7a979.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5fb725c54180a38b3ac39c85413007a51ddb3ffa --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_300_ba3e57695bcf53a7a979.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba3e57695bcf53a7a97939e58580e159f22ef4c42cfa0658b78c864a9f893fe8 +size 202714 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_1934a7ced88c6241f044.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_1934a7ced88c6241f044.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fafd87936af732c3cbe623360bb598ab3f66e317 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_1934a7ced88c6241f044.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1934a7ced88c6241f044c4edf5ffc6c3b23827a45b20499d06f3e9cca525fac0 +size 201169 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_44c371250829a86229f8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_44c371250829a86229f8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34c11a1dbf519acf1c0a0bc0ff83f4c14fb7a65c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_44c371250829a86229f8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44c371250829a86229f808766d5b4315b28f39fa163fdbc0266962fb2e4ee37c +size 232735 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_95028884d982fb8a0cb6.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_95028884d982fb8a0cb6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17f0082a2393988f0bce16584d967467e0103d11 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_95028884d982fb8a0cb6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95028884d982fb8a0cb6cd61d9d3b6d6b67aa09746fe8f9a22cc060279dbcf0f +size 175142 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_959c465e3f3e4bf8f583.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_959c465e3f3e4bf8f583.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3a9b6aa3bfff2bbdae8ce5744cdeee77eb238e95 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_959c465e3f3e4bf8f583.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:959c465e3f3e4bf8f583e0e2240cae108e11c4314439d448af58a99f5b051e5e +size 239335 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_9e7d01279fb0b2f29fa8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_9e7d01279fb0b2f29fa8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b59ef1bf5ce58f2b0dd22b9aaca0d6781b6c9677 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_9e7d01279fb0b2f29fa8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e7d01279fb0b2f29fa8eed8bd0f7625b6bd948f38beb5519336d1fbb8f3b32a +size 226432 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_9ea99436f5871458300f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_9ea99436f5871458300f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b0585dca1d4dfd916634abf17186f0fa449bfcf3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_9ea99436f5871458300f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea99436f5871458300fd8ff4c3832dd4b5156bce347018df2ce96ad4382b050 +size 186354 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_b39f29a802d827e4c574.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_b39f29a802d827e4c574.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53ea4345dcdd8c172336dcbb8003c046335cd401 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_b39f29a802d827e4c574.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39f29a802d827e4c574c195fe537efd3401775d2241401c2670f9ff59b8f717 +size 194014 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_cb6385243703cab00b9c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_cb6385243703cab00b9c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..130c01ea12138bc5c7b8568f2a5bb74c3de54e05 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_cb6385243703cab00b9c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb6385243703cab00b9cf52119d73d26d8e3b695ec115191e1fa60c94900e9c6 +size 203523 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_cbda3b15d713982cae33.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_cbda3b15d713982cae33.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b405ddce88eb7aaf9dc469cc0a570b0040215a9a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_cbda3b15d713982cae33.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbda3b15d713982cae33f565ffbe0ef806e7210bcef398d35376bcdbdf022aec +size 277844 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_d4710ab9f8ec6864adf0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_d4710ab9f8ec6864adf0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3721e78fdbeb61d9e0685f774ca6baf5366c8818 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_400_d4710ab9f8ec6864adf0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4710ab9f8ec6864adf0a6b8313b3239d07ac2e91b7910a8e72e7ccfd5a14808 +size 275804 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_091b5abe174ec3eec207.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_091b5abe174ec3eec207.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb27cb4768dd6bef30a10884cdc380328d8368cf --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_091b5abe174ec3eec207.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:091b5abe174ec3eec2079ceb00f9f0849fcd366e2e1f1cc79455c38e9c152996 +size 293684 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_270ca0f6fafd21499ef2.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_270ca0f6fafd21499ef2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..308739f8b2ee98290c6850c15b38000563866d2d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_270ca0f6fafd21499ef2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:270ca0f6fafd21499ef23cf58da504b621c62bff88025f0c20450e120b153b28 +size 211063 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_391359b72471172718e7.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_391359b72471172718e7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..97bbfee1f15a7784e5e551d79da151a0af68f5c1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_391359b72471172718e7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:391359b72471172718e740d94d1ead27ddbce45818afede53b118665f27760ac +size 239122 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_746f1da4cd90b7daec19.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_746f1da4cd90b7daec19.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11579358c97f5fcdcff01cb5094c5b5adbde5908 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_746f1da4cd90b7daec19.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:746f1da4cd90b7daec199f8cf35066f5b2166d520fc37cf17c3449fac8808270 +size 295164 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_84d3bccbc62388a00374.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_84d3bccbc62388a00374.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f50bae667ac14ee683a00ca9d4812f19744761d8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_84d3bccbc62388a00374.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84d3bccbc62388a00374ab4b9b0d926fcb4b4c90875ab4edd004de57a3dafdb2 +size 203614 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_95e3a497e5ddd6767e31.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_95e3a497e5ddd6767e31.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8cb9ec366dc2ad6259331130c0ba53e6d527093e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_95e3a497e5ddd6767e31.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95e3a497e5ddd6767e31a38eb5cc02f847c0d1d547d966e8bb0d1554f7ee3d60 +size 253240 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_ad311278e6b48ff2f590.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_ad311278e6b48ff2f590.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bde8a8199f65d4a125f1f9b418932d395dbd9331 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_ad311278e6b48ff2f590.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad311278e6b48ff2f590671224bc940c8a07de5a336596d79804ce63bc8ec38c +size 196586 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_c066d67bb8d747571584.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_c066d67bb8d747571584.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..603c5c40152b6ad7c648eea8252c5dcaceb4bf9d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_c066d67bb8d747571584.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c066d67bb8d7475715848deb7edca81135c300d210bb4288f465134f2cd5f891 +size 184635 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_c564f693bab8391cec26.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_c564f693bab8391cec26.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e49c4035bf40175541d07981f01723294c637ce0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_c564f693bab8391cec26.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c564f693bab8391cec268132a160f1a788435e4a73f258be0df913e882377887 +size 215015 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_f433153364fd254cfe02.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_f433153364fd254cfe02.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..303855071cac412a51e69b339781d03a9d58f85b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_500_f433153364fd254cfe02.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f433153364fd254cfe02054e8d763ec45a798d2f54c83f82be121a45f8902585 +size 244244 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_13dd880e1be3afbc4305.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_13dd880e1be3afbc4305.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ece3f37eb36d67cba20245df0aaa50a1e38fb45 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_13dd880e1be3afbc4305.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13dd880e1be3afbc4305b5bb6a8c99b734d6caaefe1e76344bebdb72818f1df7 +size 201215 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_1daf3ffd78eaedf00b93.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_1daf3ffd78eaedf00b93.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eebd9d6c00455d06ad8af9b64a9b8c10b3645716 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_1daf3ffd78eaedf00b93.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1daf3ffd78eaedf00b9380ee4b516b2b722a5689e49840aee293d26eb8c96d57 +size 299976 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_30ebc58e2c2585a478e2.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_30ebc58e2c2585a478e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73a644e31bd497b2d21ec631a2cf76887d14f1de --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_30ebc58e2c2585a478e2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ebc58e2c2585a478e2f07f953d3eea5395425ab91c63059b1259f80dbda6a7 +size 307007 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_5233db7fa378d994b2bf.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_5233db7fa378d994b2bf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c8f0d761d44ad4d16fae0e980ee30a40809c2f80 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_5233db7fa378d994b2bf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5233db7fa378d994b2bf6239832f9a05bf530070379eafc19084f94c3eeebe67 +size 250738 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_6213360a5c574ef5b5da.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_6213360a5c574ef5b5da.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85816fb1f3a1ed346c5c965a9d5372ba31c75ee5 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_6213360a5c574ef5b5da.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6213360a5c574ef5b5da2f23aa1bfd8d09e250309e529b2ddbe9d4f0cf8e83b4 +size 188305 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_66e24e5fe2e1c0fe4f8a.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_66e24e5fe2e1c0fe4f8a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6de0f921e9bc6f50b9b7770db26ac5aae7342ccb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_66e24e5fe2e1c0fe4f8a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66e24e5fe2e1c0fe4f8ac1ed74cf66f343a6dd2bcfa9ca17e07e81de2d8ba022 +size 217355 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_86754b3d2902e460dd2c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_86754b3d2902e460dd2c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d26b7461ccfbc49a8e2574dd124a640cfed78f30 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_86754b3d2902e460dd2c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86754b3d2902e460dd2ce4f16fd8f4e6609287a1f8f1352e19e9a9a3d801399b +size 211679 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_c634261dd6b7f89f930e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_c634261dd6b7f89f930e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..445f5dee0a788ef72e08aad8248da1eb1f1fc03c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_c634261dd6b7f89f930e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c634261dd6b7f89f930efed13fb0188a0f9139e4c89ecd5e56befd0ca715abab +size 218831 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_c7bfada6d2a58a176ceb.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_c7bfada6d2a58a176ceb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34e9f02581e35f24765bdb65e6783daf5d89ff05 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_c7bfada6d2a58a176ceb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7bfada6d2a58a176cebf022886cd2274d7f8868f2c97217b7c145efda19e73d +size 244270 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_dc0f8b7980bb0406ab08.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_dc0f8b7980bb0406ab08.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cc2905ef5daa825b7802a448d8ea2edcfc1983d1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/generated_600_dc0f8b7980bb0406ab08.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc0f8b7980bb0406ab08085cde20ebc923f209aff600cc097dbf6637b11ecb71 +size 262007 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/output.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..33b51dc31d4c6ccc7a3d740ab87b590a2a3801fa --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/output.log @@ -0,0 +1,10736 @@ +[May 1, 2026 - 12:59:38 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 1, 2026 - 12:59:38 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 1, 2026 - 12:59:38 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 1, 2026 - 12:59:38 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 1, 2026 - 12:59:38 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 1, 2026 - 12:59:38 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 1, 2026 - 12:59:38 | INFO | fastgen.trainer:run:77 ] Starting training +[May 1, 2026 - 12:59:38 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 1, 2026 - 12:59:39 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 1, 2026 - 13:00:02 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 1, 2026 - 13:00:21 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 1, 2026 - 13:00:21 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 1, 2026 - 13:00:21 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 1, 2026 - 13:00:21 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 1, 2026 - 13:01:19 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 1, 2026 - 13:01:19 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 1, 2026 - 13:01:19 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 1, 2026 - 13:01:19 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 1, 2026 - 13:01:19 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 1, 2026 - 13:01:19 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2381 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 1, 2026 - 13:01:21 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 1, 2026 - 13:01:21 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.6s +[May 1, 2026 - 13:01:21 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 1, 2026 - 13:01:21 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 1, 2026 - 13:01:24 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 1, 2026 - 13:01:24 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.4s +[May 1, 2026 - 13:01:24 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 1, 2026 - 13:01:24 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 1, 2026 - 13:01:36 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 1, 2026 - 13:01:36 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.4s +[May 1, 2026 - 13:01:36 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 1, 2026 - 13:01:36 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 1, 2026 - 13:01:36 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 1, 2026 - 13:01:36 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[May 1, 2026 - 13:01:36 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17136.85 M total params (logical). +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4285.56 M total params LOCAL on rank 0. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 1. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 2. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 3. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 1, 2026 - 13:01:43 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 1, 2026 - 13:01:43 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.184046 37.277157 37.140045 +peak_gpu_mem_gb 57.011160 57.011160 57.011160 +peak_gpu_mem_reserved_gb 59.156250 59.156250 59.156250 +util 75.500000 81.000000 70.000000 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.34 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 10.47 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 10.13 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.77 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.359684 37.452736 37.315624 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 95.750000 100.000000 88.000000 +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 46.27 seconds +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.17 +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 10.19 +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 1.57 +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:03:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.360135 37.453426 37.315849 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 92.000000 97.000000 88.000000 +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 49.31 seconds +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 13.67 +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 10.18 +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 1.59 +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:04:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.360538 37.453678 37.316109 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.211426 61.212891 61.210938 +util 96.500000 99.000000 93.000000 +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 43.66 seconds +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 10.86 +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 10.18 +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 1.57 +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:04:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.382257 37.474709 37.337986 +peak_gpu_mem_gb 96.121632 96.123194 96.121111 +peak_gpu_mem_reserved_gb 102.656738 102.658203 102.656250 +util 56.750000 65.000000 48.000000 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 162.08 seconds +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 72.09 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 8.47 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.44 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8047 iter count: 1.0 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8047 iter count: 1.0 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8047 iter count: 1.0 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8047iter count: 1.0 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:07:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.382292 37.473984 37.338139 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.820312 64.816406 +util 96.750000 100.000000 90.000000 +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 50.18 seconds +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 14.16 +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 10.14 +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 1.57 +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:08:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.382577 37.473999 37.338142 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700195 64.703125 64.699219 +util 96.000000 99.000000 89.000000 +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 48.99 seconds +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 13.57 +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 10.15 +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 1.56 +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:09:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.382749 37.474262 37.338299 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.661133 64.664062 64.660156 +util 95.750000 99.000000 92.000000 +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 48.94 seconds +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 13.56 +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 10.12 +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 1.56 +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 8-------------------- +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:09:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.382809 37.474274 37.338406 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.758789 64.761719 64.757812 +util 96.000000 100.000000 92.000000 +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 43.79 seconds +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 10.97 +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 10.13 +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 1.57 +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 9-------------------- +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:10:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.384250 37.475971 37.339478 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.259277 105.263672 105.257812 +util 66.000000 74.000000 59.000000 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 163.72 seconds +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 73.25 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 8.45 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.39 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 10-------------------- +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7822 iter count: 1.0 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7822 iter count: 1.0 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7822 iter count: 1.0 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7822iter count: 1.0 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:13:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.384321 37.476013 37.339535 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.856445 64.859375 64.855469 +util 94.500000 99.000000 90.000000 +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 47.28 seconds +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 12.72 +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 10.13 +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 1.57 +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 11-------------------- +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:14:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.384480 37.476246 37.339680 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.739258 64.742188 64.738281 +util 94.000000 99.000000 87.000000 +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 48.91 seconds +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 13.53 +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 10.13 +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 1.58 +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 12-------------------- +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:15:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.384846 37.476357 37.340729 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.778320 64.781250 64.777344 +util 97.250000 99.000000 94.000000 +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 48.93 seconds +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 13.53 +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 10.16 +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 1.58 +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 13-------------------- +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:15:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.384986 37.476398 37.340717 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.739258 64.742188 64.738281 +util 93.500000 98.000000 89.000000 +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 46.55 seconds +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 12.34 +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 10.15 +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 1.57 +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 14-------------------- +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:16:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.385214 37.476681 37.340759 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.259277 105.263672 105.257812 +util 74.250000 92.000000 61.000000 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 168.92 seconds +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 74.59 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 10.05 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.40 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 15-------------------- +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7314 iter count: 1.0 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7314 iter count: 1.0 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7314 iter count: 1.0 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7314iter count: 1.0 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:19:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.385510 37.475780 37.341656 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.973633 64.976562 64.972656 +util 96.000000 100.000000 90.000000 +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 49.44 seconds +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 12.45 +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 11.53 +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 2.17 +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 16-------------------- +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:20:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.385608 37.475895 37.341625 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.915039 64.917969 64.914062 +util 94.500000 98.000000 89.000000 +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 50.84 seconds +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 13.44 +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 10.96 +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 2.10 +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 17-------------------- +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:21:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.386096 37.475945 37.342659 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.895508 64.898438 64.894531 +util 96.750000 100.000000 92.000000 +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 52.38 seconds +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 13.57 +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 11.52 +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 2.18 +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 18-------------------- +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:22:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.386414 37.476910 37.342644 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.895508 64.898438 64.894531 +util 95.750000 100.000000 90.000000 +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 51.79 seconds +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 13.43 +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 11.48 +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 2.18 +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 19-------------------- +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:22:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.386365 37.475746 37.342922 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.298340 105.302734 105.296875 +util 74.500000 93.000000 63.000000 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 183.82 seconds +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 81.48 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 10.76 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.39 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 20-------------------- +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8643 iter count: 1.0 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8643 iter count: 1.0 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8643 iter count: 1.0 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8643iter count: 1.0 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:25:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.386655 37.476936 37.342930 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.895508 64.898438 64.894531 +util 95.500000 99.000000 88.000000 +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 50.10 seconds +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 12.57 +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 11.51 +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 2.17 +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 21-------------------- +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:26:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.386846 37.477524 37.343010 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.426758 64.429688 64.425781 +util 95.750000 99.000000 92.000000 +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 46.56 seconds +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 12.33 +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 10.15 +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 1.59 +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 22-------------------- +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:27:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387364 37.479286 37.343048 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.426758 64.429688 64.425781 +util 92.750000 99.000000 88.000000 +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 47.94 seconds +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 12.75 +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 10.17 +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 1.57 +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 23-------------------- +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:28:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387464 37.479309 37.343128 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.426758 64.429688 64.425781 +util 95.000000 99.000000 87.000000 +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 50.93 seconds +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 14.50 +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 10.15 +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 1.60 +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 24-------------------- +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:29:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387049 37.477413 37.343269 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.122559 105.126953 105.121094 +util 65.000000 69.000000 56.000000 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 160.98 seconds +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 71.87 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 8.43 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.40 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 25-------------------- +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7773 iter count: 1.0 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7773 iter count: 1.0 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7773 iter count: 1.0 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7773iter count: 1.0 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:31:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387070 37.477417 37.343292 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.836914 64.839844 64.835938 +util 95.000000 99.000000 89.000000 +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 44.86 seconds +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 11.50 +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 10.14 +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 1.57 +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 26-------------------- +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:32:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387259 37.478180 37.343292 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.719727 64.722656 64.718750 +util 93.000000 99.000000 89.000000 +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 49.22 seconds +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 13.68 +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 10.15 +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 1.57 +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 27-------------------- +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:33:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387552 37.479332 37.343304 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.661133 64.664062 64.660156 +util 94.000000 100.000000 90.000000 +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 49.00 seconds +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 13.58 +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 10.14 +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 1.57 +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 28-------------------- +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:34:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387714 37.479885 37.343304 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.685547 64.699219 64.644531 +util 94.000000 99.000000 90.000000 +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 46.21 seconds +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 12.17 +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 10.13 +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 1.58 +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 29-------------------- +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:35:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387279 37.477940 37.343342 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.210449 105.244141 105.199219 +util 67.000000 71.000000 58.000000 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 167.14 seconds +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 74.94 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 8.41 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.39 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 30-------------------- +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8213 iter count: 1.0 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8213 iter count: 1.0 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8213 iter count: 1.0 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8213iter count: 1.0 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:37:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387315 37.478035 37.343327 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.832031 64.855469 64.761719 +util 94.500000 99.000000 88.000000 +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 44.82 seconds +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 11.49 +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 10.14 +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 1.58 +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 31-------------------- +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:38:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387391 37.478302 37.343349 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.695312 64.699219 64.683594 +util 94.750000 100.000000 91.000000 +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 46.45 seconds +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 12.30 +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 10.13 +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 1.58 +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 32-------------------- +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387780 37.479820 37.343353 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700195 64.703125 64.699219 +util 96.250000 99.000000 89.000000 +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 43.72 seconds +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 10.94 +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 10.16 +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 1.57 +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 33-------------------- +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0060 iter count: 1.0 +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:40:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387782 37.479824 37.343353 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.788086 64.816406 64.703125 +util 92.750000 97.000000 89.000000 +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 48.89 seconds +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 13.52 +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 10.13 +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 1.58 +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 34-------------------- +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:40:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387663 37.479286 37.343361 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.186035 105.205078 105.179688 +util 61.750000 66.000000 58.000000 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 161.81 seconds +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 72.27 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 8.48 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.40 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 35-------------------- +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7324 iter count: 1.0 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7324 iter count: 1.0 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7324 iter count: 1.0 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7324iter count: 1.0 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:43:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387677 37.479290 37.343403 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.822754 64.835938 64.783203 +util 93.750000 99.000000 90.000000 +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 47.57 seconds +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 12.87 +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 10.12 +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 1.57 +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 36-------------------- +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:44:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387708 37.479404 37.343380 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.715332 64.718750 64.705078 +util 96.500000 99.000000 90.000000 +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 43.73 seconds +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 10.94 +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 10.13 +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 1.57 +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 37-------------------- +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:45:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387820 37.479603 37.343361 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700684 64.705078 64.699219 +util 93.750000 99.000000 90.000000 +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 49.06 seconds +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 13.62 +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 10.13 +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 1.56 +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 38-------------------- +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:45:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387852 37.479626 37.343357 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.695801 64.744141 64.679688 +util 93.750000 99.000000 90.000000 +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 46.40 seconds +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 12.27 +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 10.14 +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 1.56 +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 39-------------------- +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:46:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387884 37.479660 37.343357 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.156738 105.263672 105.121094 +util 66.750000 75.000000 58.000000 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 161.78 seconds +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 72.30 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 8.42 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.39 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 40-------------------- +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7568 iter count: 1.0 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7568 iter count: 1.0 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7568 iter count: 1.0 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7568iter count: 1.0 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:49:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387708 37.478954 37.343357 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.876465 64.939453 64.855469 +util 96.750000 100.000000 93.000000 +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 50.09 seconds +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 14.12 +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 10.13 +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 1.58 +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 41-------------------- +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:50:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387714 37.478954 37.343357 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.744629 64.835938 64.470703 +util 97.250000 99.000000 94.000000 +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 46.41 seconds +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 12.27 +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 10.15 +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 1.58 +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 42-------------------- +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:50:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387775 37.479183 37.343361 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.808105 64.958984 64.757812 +util 94.500000 99.000000 88.000000 +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 43.69 seconds +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 10.92 +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 10.14 +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 1.56 +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 43-------------------- +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:51:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387775 37.479183 37.343361 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.651855 64.718750 64.451172 +util 95.000000 99.000000 91.000000 +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 46.28 seconds +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 12.22 +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 10.14 +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 1.57 +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 44-------------------- +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:52:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387883 37.479519 37.343445 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.293457 105.355469 105.107422 +util 64.500000 73.000000 56.000000 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 161.71 seconds +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 72.25 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 8.42 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.40 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 45-------------------- +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7480 iter count: 1.0 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7480 iter count: 1.0 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7480 iter count: 1.0 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7480iter count: 1.0 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:55:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387890 37.479523 37.343445 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.837402 64.953125 64.490234 +util 92.000000 99.000000 88.000000 +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 50.41 seconds +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 14.27 +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 10.16 +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 1.58 +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 46-------------------- +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:56:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387955 37.479763 37.343445 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.788574 64.894531 64.470703 +util 92.250000 99.000000 88.000000 +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 46.40 seconds +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 12.27 +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 10.14 +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 1.57 +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 47-------------------- +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:56:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387977 37.479767 37.343529 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.744629 64.939453 64.679688 +util 92.250000 99.000000 88.000000 +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 49.00 seconds +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 13.59 +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 10.11 +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 1.55 +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 48-------------------- +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388109 37.480263 37.343513 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.729980 64.939453 64.660156 +util 93.750000 100.000000 86.000000 +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 49.17 seconds +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 13.66 +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 10.14 +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 1.57 +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 49-------------------- +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 13:58:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388084 37.480091 37.343555 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.186035 105.199219 105.146484 +util 64.000000 67.000000 61.000000 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 158.71 seconds +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 70.76 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 8.42 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.39 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 50-------------------- +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7568 iter count: 1.0 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7568 iter count: 1.0 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7568 iter count: 1.0 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7568iter count: 1.0 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:01:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388093 37.480072 37.343555 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.783203 64.859375 64.757812 +util 96.250000 99.000000 93.000000 +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 47.41 seconds +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 12.79 +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 10.13 +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 1.58 +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 51-------------------- +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:01:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388088 37.480072 37.343578 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.670898 64.703125 64.660156 +util 96.500000 100.000000 93.000000 +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 46.41 seconds +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 12.28 +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 10.12 +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 1.58 +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 52-------------------- +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:02:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388092 37.480076 37.343575 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.670898 64.703125 64.660156 +util 93.000000 99.000000 89.000000 +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 46.52 seconds +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 12.34 +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 10.14 +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 1.57 +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 53-------------------- +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:03:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388287 37.480831 37.343601 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.719727 64.722656 64.718750 +util 91.000000 95.000000 87.000000 +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 46.21 seconds +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 12.19 +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 10.13 +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 1.57 +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 54-------------------- +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:04:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388005 37.479706 37.343582 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.312988 105.335938 105.244141 +util 64.000000 72.000000 56.000000 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 161.54 seconds +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 72.17 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 8.43 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.39 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 55-------------------- +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7285 iter count: 1.0 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7285 iter count: 1.0 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7285 iter count: 1.0 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7285iter count: 1.0 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:06:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388011 37.479702 37.343582 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.529297 64.781250 64.445312 +util 93.750000 100.000000 90.000000 +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 44.84 seconds +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 11.49 +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 10.13 +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 1.57 +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 56-------------------- +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:07:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388086 37.479961 37.343597 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.812500 64.855469 64.683594 +util 91.750000 96.000000 88.000000 +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 49.04 seconds +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 13.58 +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 10.14 +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 1.59 +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 57-------------------- +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:08:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388078 37.479961 37.343567 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.514648 64.664062 64.464844 +util 95.500000 99.000000 92.000000 +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 46.43 seconds +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 12.29 +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 10.13 +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 1.57 +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 58-------------------- +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:09:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388213 37.480461 37.343559 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.519531 64.742188 64.445312 +util 96.000000 100.000000 88.000000 +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 46.46 seconds +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 12.31 +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 10.13 +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 1.56 +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 59-------------------- +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:09:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388020 37.479637 37.343567 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.142090 105.205078 105.121094 +util 66.750000 70.000000 63.000000 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 161.76 seconds +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 72.28 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 8.41 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.39 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 60-------------------- +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7314 iter count: 1.0 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7314 iter count: 1.0 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7314 iter count: 1.0 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7314iter count: 1.0 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:12:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388021 37.479649 37.343567 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.632324 64.841797 64.562500 +util 93.750000 99.000000 87.000000 +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 50.27 seconds +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 14.21 +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 10.12 +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 1.57 +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 61-------------------- +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:13:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388019 37.479656 37.343567 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.598145 64.763672 64.542969 +util 93.000000 100.000000 85.000000 +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 46.40 seconds +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 12.27 +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 10.14 +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 1.59 +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 62-------------------- +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:14:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388018 37.479656 37.343567 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.568848 64.822266 64.484375 +util 93.250000 100.000000 89.000000 +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 43.65 seconds +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 10.89 +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 10.14 +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 1.56 +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 63-------------------- +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:15:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388060 37.479679 37.343594 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.612793 64.822266 64.542969 +util 91.250000 95.000000 87.000000 +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 46.42 seconds +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 12.28 +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 10.15 +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 1.57 +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 64-------------------- +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:15:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387983 37.479359 37.343578 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.181152 105.185547 105.179688 +util 66.000000 69.000000 63.000000 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 164.47 seconds +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 73.62 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 8.42 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.40 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 65-------------------- +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6963 iter count: 1.0 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6963 iter count: 1.0 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6963 iter count: 1.0 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6963iter count: 1.0 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:18:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387977 37.479328 37.343578 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 64.914062 64.859375 +util 94.250000 100.000000 88.000000 +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 47.46 seconds +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 12.81 +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 10.13 +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 1.56 +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 66-------------------- +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:19:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387982 37.479340 37.343578 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.866211 64.875000 64.839844 +util 95.750000 100.000000 87.000000 +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 46.38 seconds +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 12.26 +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 10.14 +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 1.56 +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 67-------------------- +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:20:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388067 37.479591 37.343658 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.851562 64.855469 64.839844 +util 95.000000 99.000000 86.000000 +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 49.00 seconds +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 13.58 +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 10.15 +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 1.57 +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 68-------------------- +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:20:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388067 37.479568 37.343681 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.880859 64.894531 64.839844 +util 94.750000 99.000000 89.000000 +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 49.06 seconds +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 13.59 +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 10.15 +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 1.57 +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 69-------------------- +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:21:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387893 37.478870 37.343658 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.161621 105.224609 105.140625 +util 63.000000 72.000000 58.000000 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 161.84 seconds +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 72.30 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 8.42 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.40 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 70-------------------- +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7168 iter count: 1.0 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7168 iter count: 1.0 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7168 iter count: 1.0 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7168iter count: 1.0 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:24:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387899 37.478855 37.343681 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.812500 64.859375 64.796875 +util 92.500000 96.000000 89.000000 +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 47.62 seconds +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 12.90 +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 10.14 +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 1.56 +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 71-------------------- +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387977 37.479122 37.343712 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.797852 64.859375 64.777344 +util 95.500000 99.000000 92.000000 +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 46.24 seconds +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 12.20 +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 10.14 +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 1.57 +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 72-------------------- +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:26:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387985 37.479122 37.343731 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.729492 64.820312 64.699219 +util 92.750000 97.000000 89.000000 +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 48.97 seconds +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 13.56 +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 10.13 +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 1.57 +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 73-------------------- +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:26:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388126 37.479622 37.343731 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.763672 64.839844 64.738281 +util 94.750000 99.000000 91.000000 +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 49.03 seconds +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 13.58 +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 10.15 +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 1.56 +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 74-------------------- +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:27:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387965 37.478905 37.343746 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.200684 105.263672 105.179688 +util 60.500000 70.000000 55.000000 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 161.33 seconds +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 72.03 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 8.43 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.40 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 75-------------------- +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6602 iter count: 1.0 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6602 iter count: 1.0 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6602 iter count: 1.0 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6602iter count: 1.0 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:30:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.387965 37.478909 37.343769 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.841797 64.859375 64.835938 +util 93.500000 97.000000 90.000000 +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 50.26 seconds +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 14.21 +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 10.14 +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 1.57 +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 76-------------------- +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:31:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388096 37.479424 37.343769 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734375 64.757812 64.664062 +util 92.000000 99.000000 88.000000 +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 48.94 seconds +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 13.56 +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 10.12 +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 1.56 +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 77-------------------- +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:31:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388217 37.479919 37.343731 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.680664 64.742188 64.660156 +util 98.000000 100.000000 95.000000 +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 49.12 seconds +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 13.62 +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 10.14 +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 1.58 +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 78-------------------- +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:32:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388288 37.480179 37.343750 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.724609 64.742188 64.718750 +util 94.250000 98.000000 90.000000 +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 49.16 seconds +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 13.64 +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 10.15 +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 1.59 +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 79-------------------- +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:33:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388103 37.479397 37.343773 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.200684 105.263672 105.179688 +util 59.500000 62.000000 56.000000 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 156.35 seconds +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 69.58 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 8.41 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.40 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 80-------------------- +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7109 iter count: 1.0 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7109 iter count: 1.0 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7109 iter count: 1.0 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7109iter count: 1.0 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:36:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388105 37.479397 37.343761 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.807617 64.816406 64.781250 +util 93.750000 99.000000 87.000000 +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 44.76 seconds +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 11.46 +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 10.13 +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 1.57 +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 81-------------------- +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:36:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388251 37.479904 37.343781 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.744141 64.757812 64.703125 +util 95.500000 99.000000 92.000000 +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 43.70 seconds +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 10.93 +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 10.12 +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 1.58 +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 82-------------------- +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:37:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388261 37.479904 37.343758 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.783203 64.816406 64.683594 +util 96.250000 99.000000 93.000000 +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 46.42 seconds +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 12.29 +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 10.13 +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 1.57 +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 83-------------------- +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:38:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388322 37.480171 37.343758 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734375 64.738281 64.722656 +util 91.500000 95.000000 88.000000 +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 46.34 seconds +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 12.25 +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 10.12 +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 1.57 +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 84-------------------- +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:39:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388157 37.479473 37.343765 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.176270 105.224609 105.160156 +util 61.750000 65.000000 58.000000 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 161.22 seconds +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 71.99 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 8.45 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.39 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 85-------------------- +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6709 iter count: 1.0 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6709 iter count: 1.0 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6709 iter count: 1.0 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6709iter count: 1.0 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:41:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388178 37.479473 37.343765 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734375 64.816406 64.488281 +util 97.000000 100.000000 90.000000 +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 47.51 seconds +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 12.82 +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 10.13 +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 1.58 +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 86-------------------- +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:42:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388251 37.479767 37.343765 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.607422 64.640625 64.507812 +util 91.500000 95.000000 88.000000 +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 46.47 seconds +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 12.30 +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 10.14 +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 1.58 +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 87-------------------- +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:43:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388618 37.481281 37.343765 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.587891 64.640625 64.429688 +util 93.500000 99.000000 90.000000 +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 46.38 seconds +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 12.26 +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 10.14 +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 1.56 +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 88-------------------- +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:44:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388741 37.481762 37.343765 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.729492 64.937500 64.660156 +util 92.500000 96.000000 89.000000 +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 46.23 seconds +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 12.20 +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 10.12 +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 1.56 +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 89-------------------- +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:45:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388227 37.479702 37.343765 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.317871 105.322266 105.316406 +util 65.500000 73.000000 57.000000 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 164.37 seconds +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 73.56 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 8.43 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.40 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 90-------------------- +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7080 iter count: 1.0 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7080 iter count: 1.0 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7080 iter count: 1.0 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7080iter count: 1.0 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:47:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388222 37.479706 37.343765 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.841797 64.917969 64.816406 +util 93.750000 99.000000 87.000000 +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 44.77 seconds +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 11.46 +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 10.14 +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 1.57 +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 91-------------------- +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:48:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388410 37.480450 37.343765 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.739258 64.917969 64.679688 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 49.15 seconds +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 13.65 +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 10.13 +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 1.58 +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 92-------------------- +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:49:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388737 37.481716 37.343800 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.739258 64.835938 64.449219 +util 93.000000 100.000000 89.000000 +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 46.27 seconds +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 12.21 +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 10.15 +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 1.56 +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 93-------------------- +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:50:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388734 37.481693 37.343811 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.695312 64.777344 64.449219 +util 96.000000 100.000000 87.000000 +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 43.74 seconds +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 10.93 +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 10.16 +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 1.56 +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 94-------------------- +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:50:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388664 37.481415 37.343796 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.181152 105.244141 105.160156 +util 66.750000 70.000000 63.000000 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 158.77 seconds +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 70.79 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 8.42 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.39 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 95-------------------- +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7393 iter count: 1.0 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7393 iter count: 1.0 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7393 iter count: 1.0 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7393iter count: 1.0 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:53:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388600 37.480900 37.343925 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.563477 64.917969 64.445312 +util 95.500000 99.000000 92.000000 +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 44.76 seconds +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 11.45 +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 10.14 +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 1.57 +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 96-------------------- +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:54:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388599 37.480877 37.343925 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.568359 64.937500 64.445312 +util 92.750000 99.000000 89.000000 +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 46.33 seconds +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.25 +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 10.13 +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 1.57 +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 97-------------------- +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:55:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388603 37.480877 37.343948 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.933594 64.468750 +util 92.750000 99.000000 89.000000 +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 46.20 seconds +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.20 +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 10.12 +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 1.56 +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 98-------------------- +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:55:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388595 37.480877 37.343925 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.431641 64.449219 64.425781 +util 92.500000 96.000000 89.000000 +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 46.43 seconds +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 12.31 +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 10.13 +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 1.57 +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 99-------------------- +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:56:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.388597 37.480877 37.343929 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.225098 105.244141 105.218750 +util 59.500000 63.000000 56.000000 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 166.98 seconds +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 74.87 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 8.41 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.40 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 100-------------------- +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7236 iter count: 1.0 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7236 iter count: 1.0 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7236 iter count: 1.0 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7236iter count: 1.0 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 14:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 14:59:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 18.10 seconds +[May 1, 2026 - 14:59:38 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[May 1, 2026 - 14:59:38 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000100 +[May 1, 2026 - 15:00:18 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 100 +[May 1, 2026 - 15:00:18 | INFO | fastgen.trainer:validate:358 ] Validation iteration 100 +[May 1, 2026 - 15:00:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:00:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 1, 2026 - 15:00:24 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 1, 2026 - 15:00:25 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.496 +[May 1, 2026 - 15:00:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:00:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:00:30 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.929 +[May 1, 2026 - 15:00:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:00:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:00:35 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.114 +[May 1, 2026 - 15:00:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:00:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:00:40 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.293 +[May 1, 2026 - 15:00:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:00:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:00:45 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.914 +[May 1, 2026 - 15:00:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:00:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:00:50 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.571 +[May 1, 2026 - 15:00:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:00:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:00:56 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.004 +[May 1, 2026 - 15:00:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:00:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:01:01 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.755 +[May 1, 2026 - 15:01:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:01:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:01:06 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.384 +[May 1, 2026 - 15:01:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 15:01:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 15:01:11 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.804 +[May 1, 2026 - 15:01:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 100-------------------- +[May 1, 2026 - 15:01:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 15:01:27 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 100 +[May 1, 2026 - 15:01:27 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.226 (n=10) +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228738 41.348660 41.174606 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.967773 68.222656 67.882812 +util 93.250000 100.000000 89.000000 +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 101 : avg iteration time 170.37 seconds +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 101 : data loading time 0.00 +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 101 : avg forward pass time 10.53 +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 101 : backward pass time 10.15 +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 101 : optimizer step time 1.57 +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 101-------------------- +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:02:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228791 41.348660 41.174801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.914062 68.242188 67.804688 +util 94.250000 97.000000 91.000000 +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 102 : avg iteration time 49.25 seconds +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 102 : data loading time 0.00 +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 102 : avg forward pass time 13.67 +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 102 : backward pass time 10.18 +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 102 : optimizer step time 1.56 +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 102-------------------- +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:03:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228541 41.347538 41.174778 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.904297 68.203125 67.804688 +util 91.750000 99.000000 88.000000 +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 103 : avg iteration time 43.94 seconds +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 103 : data loading time 0.00 +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 103 : avg forward pass time 11.01 +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 103 : backward pass time 10.17 +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 103 : optimizer step time 1.57 +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 103-------------------- +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:03:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228307 41.346577 41.174801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.933594 68.203125 67.843750 +util 92.250000 100.000000 88.000000 +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 104 : avg iteration time 46.59 seconds +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 104 : data loading time 0.00 +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 104 : avg forward pass time 12.35 +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 104 : backward pass time 10.14 +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 104 : optimizer step time 1.58 +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 104-------------------- +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:04:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229047 41.345905 41.175995 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.609375 107.603516 +util 64.250000 73.000000 55.000000 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 105 : avg iteration time 161.59 seconds +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 105 : data loading time 0.00 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 105 : avg forward pass time 72.17 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 105 : backward pass time 8.42 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 105 : optimizer step time 0.40 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 105-------------------- +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7139 iter count: 1.0 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7139 iter count: 1.0 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7139 iter count: 1.0 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7139iter count: 1.0 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:07:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228885 41.345184 41.176006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.679688 67.785156 67.363281 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 106 : avg iteration time 45.15 seconds +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 106 : data loading time 0.00 +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 106 : avg forward pass time 11.65 +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 106 : backward pass time 10.12 +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 106 : optimizer step time 1.56 +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 106-------------------- +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:07:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228909 41.345284 41.176006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.655273 67.765625 67.324219 +util 92.750000 100.000000 89.000000 +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 107 : avg iteration time 49.26 seconds +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 107 : data loading time 0.00 +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 107 : avg forward pass time 13.72 +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 107 : backward pass time 10.14 +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 107 : optimizer step time 1.56 +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 107-------------------- +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:08:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228904 41.345284 41.176006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.679688 67.765625 67.421875 +util 97.250000 99.000000 94.000000 +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 108 : avg iteration time 49.11 seconds +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 108 : data loading time 0.00 +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 108 : avg forward pass time 13.62 +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 108 : backward pass time 10.14 +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 108 : optimizer step time 1.58 +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 108-------------------- +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:09:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228997 41.345581 41.175983 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.460938 67.453125 +util 94.000000 97.000000 91.000000 +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 109 : avg iteration time 46.35 seconds +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 109 : data loading time 0.00 +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 109 : avg forward pass time 12.24 +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 109 : backward pass time 10.14 +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 109 : optimizer step time 1.58 +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 109-------------------- +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:10:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228857 41.345001 41.175983 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.658691 107.707031 107.642578 +util 65.500000 74.000000 57.000000 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 110 : avg iteration time 161.95 seconds +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 110 : data loading time 0.00 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 110 : avg forward pass time 72.34 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 110 : backward pass time 8.42 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 110 : optimizer step time 0.41 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 110-------------------- +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7158 iter count: 1.0 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7158 iter count: 1.0 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7158 iter count: 1.0 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7158iter count: 1.0 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:13:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228874 41.345005 41.176006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373535 67.376953 67.363281 +util 96.250000 99.000000 93.000000 +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 111 : avg iteration time 47.68 seconds +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 111 : data loading time 0.00 +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 111 : avg forward pass time 12.91 +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 111 : backward pass time 10.14 +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 111 : optimizer step time 1.57 +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 111-------------------- +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:13:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228878 41.345016 41.175987 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.455078 67.421875 +util 92.250000 95.000000 89.000000 +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 112 : avg iteration time 46.34 seconds +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 112 : data loading time 0.00 +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 112 : avg forward pass time 12.24 +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 112 : backward pass time 10.13 +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 112 : optimizer step time 1.58 +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 112-------------------- +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:14:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228890 41.345005 41.176006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.485840 67.519531 67.474609 +util 97.500000 99.000000 95.000000 +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 113 : avg iteration time 43.66 seconds +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 113 : data loading time 0.00 +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 113 : avg forward pass time 10.91 +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 113 : backward pass time 10.16 +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 113 : optimizer step time 1.56 +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 113-------------------- +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:15:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229148 41.345989 41.176006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.466309 67.474609 67.441406 +util 94.250000 99.000000 88.000000 +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 114 : avg iteration time 43.69 seconds +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 114 : data loading time 0.00 +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 114 : avg forward pass time 10.91 +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 114 : backward pass time 10.13 +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 114 : optimizer step time 1.56 +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 114-------------------- +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:16:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229039 41.345234 41.176064 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.609375 107.603516 +util 60.750000 63.000000 58.000000 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 115 : avg iteration time 164.65 seconds +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 115 : data loading time 0.00 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 115 : avg forward pass time 73.71 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 115 : backward pass time 8.42 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 115 : optimizer step time 0.40 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 115-------------------- +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6904 iter count: 1.0 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6904 iter count: 1.0 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6904 iter count: 1.0 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6904iter count: 1.0 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:18:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229057 41.345257 41.176113 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.357422 67.421875 67.335938 +util 93.750000 99.000000 87.000000 +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 116 : avg iteration time 50.33 seconds +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 116 : data loading time 0.00 +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 116 : avg forward pass time 14.23 +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 116 : backward pass time 10.15 +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 116 : optimizer step time 1.57 +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 116-------------------- +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:19:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229114 41.345482 41.176113 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.519531 67.335938 +util 93.750000 99.000000 87.000000 +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 117 : avg iteration time 43.80 seconds +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 117 : data loading time 0.00 +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 117 : avg forward pass time 10.98 +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 117 : backward pass time 10.12 +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 117 : optimizer step time 1.56 +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 117-------------------- +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:20:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229139 41.345490 41.176113 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.578125 67.316406 +util 90.500000 94.000000 87.000000 +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 118 : avg iteration time 43.65 seconds +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 118 : data loading time 0.00 +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 118 : avg forward pass time 10.89 +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 118 : backward pass time 10.14 +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 118 : optimizer step time 1.57 +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 118-------------------- +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:21:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229142 41.345482 41.176113 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.142578 67.441406 67.042969 +util 96.000000 99.000000 89.000000 +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 119 : avg iteration time 46.31 seconds +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 119 : data loading time 0.00 +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 119 : avg forward pass time 12.23 +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 119 : backward pass time 10.14 +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 119 : optimizer step time 1.56 +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 119-------------------- +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:21:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228915 41.344570 41.176113 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.746094 107.525391 +util 62.750000 71.000000 54.000000 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 120 : avg iteration time 164.04 seconds +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 120 : data loading time 0.00 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 120 : avg forward pass time 73.40 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 120 : backward pass time 8.42 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 120 : optimizer step time 0.40 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 120-------------------- +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7334 iter count: 1.0 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7334 iter count: 1.0 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7334 iter count: 1.0 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7334iter count: 1.0 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:24:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229035 41.345051 41.176113 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.333008 67.382812 67.316406 +util 96.750000 99.000000 94.000000 +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 121 : avg iteration time 44.72 seconds +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 121 : data loading time 0.00 +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 121 : avg forward pass time 11.43 +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 121 : backward pass time 10.15 +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 121 : optimizer step time 1.58 +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 121-------------------- +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:25:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229038 41.345051 41.176113 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.245117 67.500000 67.160156 +util 98.500000 100.000000 96.000000 +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 122 : avg iteration time 46.43 seconds +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 122 : data loading time 0.00 +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 122 : avg forward pass time 12.29 +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 122 : backward pass time 10.14 +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 122 : optimizer step time 1.58 +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 122-------------------- +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:26:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:26:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229033 41.345051 41.176113 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.500000 67.296875 +util 96.750000 100.000000 88.000000 +[May 1, 2026 - 15:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 123 : avg iteration time 46.63 seconds +[May 1, 2026 - 15:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 123 : data loading time 0.00 +[May 1, 2026 - 15:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 123 : avg forward pass time 12.39 +[May 1, 2026 - 15:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 123 : backward pass time 10.13 +[May 1, 2026 - 15:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 123 : optimizer step time 1.57 +[May 1, 2026 - 15:26:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 123-------------------- +[May 1, 2026 - 15:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 15:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 15:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:26:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229281 41.346046 41.176113 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.500000 67.316406 +util 97.250000 99.000000 95.000000 +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 124 : avg iteration time 46.35 seconds +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 124 : data loading time 0.00 +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 124 : avg forward pass time 12.26 +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 124 : backward pass time 10.14 +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 124 : optimizer step time 1.56 +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 124-------------------- +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:27:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229406 41.346409 41.176113 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.785156 107.583984 +util 61.250000 74.000000 54.000000 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 125 : avg iteration time 159.33 seconds +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 125 : data loading time 0.00 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 125 : avg forward pass time 71.00 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 125 : backward pass time 8.43 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 125 : optimizer step time 0.44 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 125-------------------- +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6074 iter count: 1.0 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6074 iter count: 1.0 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6074 iter count: 1.0 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6074iter count: 1.0 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:30:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229431 41.346466 41.176174 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.314941 67.318359 67.304688 +util 98.500000 100.000000 96.000000 +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 126 : avg iteration time 50.29 seconds +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 126 : data loading time 0.00 +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 126 : avg forward pass time 14.22 +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 126 : backward pass time 10.13 +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 126 : optimizer step time 1.57 +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 126-------------------- +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:31:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229420 41.346420 41.176174 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.349121 67.357422 67.324219 +util 97.750000 99.000000 95.000000 +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 127 : avg iteration time 43.79 seconds +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 127 : data loading time 0.00 +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 127 : avg forward pass time 10.96 +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 127 : backward pass time 10.14 +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 127 : optimizer step time 1.57 +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 127-------------------- +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:31:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229445 41.346470 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.329590 67.421875 67.298828 +util 92.500000 99.000000 89.000000 +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 128 : avg iteration time 43.90 seconds +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 128 : data loading time 0.00 +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 128 : avg forward pass time 11.00 +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 128 : backward pass time 10.19 +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 128 : optimizer step time 1.57 +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 128-------------------- +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:32:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229505 41.346706 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358887 67.480469 67.318359 +util 91.000000 99.000000 87.000000 +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 129 : avg iteration time 43.78 seconds +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 129 : data loading time 0.00 +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 129 : avg forward pass time 10.96 +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 129 : backward pass time 10.14 +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 129 : optimizer step time 1.57 +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 129-------------------- +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:33:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229273 41.345760 41.176186 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.628906 107.623047 +util 65.500000 68.000000 63.000000 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 130 : avg iteration time 162.11 seconds +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 130 : data loading time 0.00 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 130 : avg forward pass time 72.46 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 130 : backward pass time 8.42 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 130 : optimizer step time 0.40 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 130-------------------- +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6719 iter count: 1.0 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6719 iter count: 1.0 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6719 iter count: 1.0 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6719iter count: 1.0 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:36:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229258 41.345715 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.582031 68.027344 67.433594 +util 91.250000 94.000000 88.000000 +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 131 : avg iteration time 50.53 seconds +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 131 : data loading time 0.00 +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 131 : avg forward pass time 14.33 +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 131 : backward pass time 10.15 +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 131 : optimizer step time 1.57 +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 131-------------------- +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:36:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229295 41.345814 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.636719 67.355469 +util 99.250000 100.000000 98.000000 +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 132 : avg iteration time 46.51 seconds +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 132 : data loading time 0.00 +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 132 : avg forward pass time 12.32 +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 132 : backward pass time 10.14 +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 132 : optimizer step time 1.58 +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 132-------------------- +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:37:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229339 41.345963 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.578125 67.394531 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 133 : avg iteration time 46.59 seconds +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 133 : data loading time 0.00 +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 133 : avg forward pass time 12.35 +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 133 : backward pass time 10.17 +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 133 : optimizer step time 1.58 +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 133-------------------- +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:38:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229344 41.345963 41.176208 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.597656 67.335938 +util 95.250000 98.000000 92.000000 +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 134 : avg iteration time 49.15 seconds +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 134 : data loading time 0.00 +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 134 : avg forward pass time 13.64 +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 134 : backward pass time 10.16 +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 134 : optimizer step time 1.56 +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 134-------------------- +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:39:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229173 41.345131 41.176186 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.648438 107.525391 +util 67.250000 72.000000 57.000000 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 135 : avg iteration time 158.89 seconds +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 135 : data loading time 0.00 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 135 : avg forward pass time 70.86 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 135 : backward pass time 8.41 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 135 : optimizer step time 0.40 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 135-------------------- +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6709 iter count: 1.0 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6709 iter count: 1.0 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6709 iter count: 1.0 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6709iter count: 1.0 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:41:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229191 41.345139 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.357422 67.421875 67.335938 +util 96.250000 100.000000 87.000000 +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 136 : avg iteration time 47.58 seconds +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 136 : data loading time 0.00 +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 136 : avg forward pass time 12.86 +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 136 : backward pass time 10.13 +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 136 : optimizer step time 1.57 +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 136-------------------- +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:42:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229287 41.345524 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.480469 67.335938 +util 93.250000 99.000000 90.000000 +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 137 : avg iteration time 46.47 seconds +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 137 : data loading time 0.00 +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 137 : avg forward pass time 12.32 +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 137 : backward pass time 10.13 +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 137 : optimizer step time 1.56 +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 137-------------------- +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:43:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229356 41.345757 41.176224 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.416016 67.597656 67.355469 +util 92.500000 99.000000 89.000000 +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 138 : avg iteration time 46.39 seconds +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 138 : data loading time 0.00 +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 138 : avg forward pass time 12.27 +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 138 : backward pass time 10.14 +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 138 : optimizer step time 1.57 +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 138-------------------- +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:44:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229435 41.346115 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.259766 67.500000 67.179688 +util 98.500000 100.000000 96.000000 +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 139 : avg iteration time 46.31 seconds +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 139 : data loading time 0.00 +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 139 : avg forward pass time 12.24 +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 139 : backward pass time 10.14 +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 139 : optimizer step time 1.57 +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 139-------------------- +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:45:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228923 41.344032 41.176201 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.648438 107.564453 +util 65.750000 70.000000 56.000000 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 140 : avg iteration time 164.24 seconds +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 140 : data loading time 0.00 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 140 : avg forward pass time 73.51 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 140 : backward pass time 8.43 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 140 : optimizer step time 0.40 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 140-------------------- +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7012 iter count: 1.0 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7012 iter count: 1.0 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7012 iter count: 1.0 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7012iter count: 1.0 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:47:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228914 41.344032 41.176186 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.503906 67.832031 67.394531 +util 91.500000 94.000000 89.000000 +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 141 : avg iteration time 47.48 seconds +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 141 : data loading time 0.00 +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 141 : avg forward pass time 12.81 +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 141 : backward pass time 10.13 +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 141 : optimizer step time 1.57 +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 141-------------------- +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:48:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229121 41.344746 41.176281 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.597656 67.316406 +util 93.750000 99.000000 87.000000 +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 142 : avg iteration time 46.55 seconds +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 142 : data loading time 0.00 +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 142 : avg forward pass time 12.35 +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 142 : backward pass time 10.12 +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 142 : optimizer step time 1.58 +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 142-------------------- +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:49:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229142 41.344883 41.176247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.342773 67.421875 67.316406 +util 91.250000 94.000000 88.000000 +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 143 : avg iteration time 46.45 seconds +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 143 : data loading time 0.00 +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 143 : avg forward pass time 12.28 +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 143 : backward pass time 10.17 +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 143 : optimizer step time 1.57 +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 143-------------------- +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:50:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229224 41.345192 41.176258 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.376953 67.617188 67.296875 +util 92.000000 100.000000 88.000000 +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 144 : avg iteration time 43.77 seconds +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 144 : data loading time 0.00 +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 144 : avg forward pass time 10.95 +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 144 : backward pass time 10.14 +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 144 : optimizer step time 1.57 +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 144-------------------- +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:50:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228907 41.343880 41.176270 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.687500 107.642578 +util 65.750000 68.000000 63.000000 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 145 : avg iteration time 161.70 seconds +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 145 : data loading time 0.00 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 145 : avg forward pass time 72.20 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 145 : backward pass time 8.41 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 145 : optimizer step time 0.39 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 145-------------------- +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6758 iter count: 1.0 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6758 iter count: 1.0 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6758 iter count: 1.0 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6758iter count: 1.0 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:53:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228905 41.343880 41.176247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.416016 67.832031 67.277344 +util 93.500000 99.000000 87.000000 +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 146 : avg iteration time 47.67 seconds +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 146 : data loading time 0.00 +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 146 : avg forward pass time 12.91 +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 146 : backward pass time 10.14 +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 146 : optimizer step time 1.57 +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 146-------------------- +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:54:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229143 41.344810 41.176292 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.420898 67.675781 67.335938 +util 94.000000 97.000000 91.000000 +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 147 : avg iteration time 49.29 seconds +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 147 : data loading time 0.00 +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 147 : avg forward pass time 13.72 +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 147 : backward pass time 10.13 +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 147 : optimizer step time 1.59 +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 147-------------------- +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:55:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229132 41.344810 41.176247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.753906 67.316406 +util 93.500000 99.000000 87.000000 +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 148 : avg iteration time 46.53 seconds +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 148 : data loading time 0.00 +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 148 : avg forward pass time 12.36 +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 148 : backward pass time 10.12 +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 148 : optimizer step time 1.57 +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 148-------------------- +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:55:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229194 41.345032 41.176270 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.695312 67.316406 +util 97.000000 99.000000 94.000000 +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 149 : avg iteration time 43.71 seconds +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 149 : data loading time 0.00 +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 149 : avg forward pass time 10.91 +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 149 : backward pass time 10.14 +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 149 : optimizer step time 1.57 +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 149-------------------- +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:56:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228815 41.343521 41.176270 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.722168 107.785156 107.701172 +util 62.500000 65.000000 60.000000 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 150 : avg iteration time 161.56 seconds +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 150 : data loading time 0.00 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 150 : avg forward pass time 72.17 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 150 : backward pass time 8.41 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 150 : optimizer step time 0.39 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 150-------------------- +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6523 iter count: 1.0 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6523 iter count: 1.0 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6523 iter count: 1.0 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6523iter count: 1.0 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 15:59:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228814 41.343540 41.176247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.480469 67.277344 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 151 : avg iteration time 47.47 seconds +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 151 : data loading time 0.00 +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 151 : avg forward pass time 12.81 +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 151 : backward pass time 10.14 +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 151 : optimizer step time 1.58 +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 151-------------------- +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:00:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229046 41.344444 41.176270 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.342773 67.480469 67.296875 +util 93.750000 100.000000 87.000000 +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 152 : avg iteration time 49.05 seconds +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 152 : data loading time 0.00 +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 152 : avg forward pass time 13.60 +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 152 : backward pass time 10.15 +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 152 : optimizer step time 1.56 +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 152-------------------- +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:00:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229406 41.345909 41.176247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.220703 67.402344 67.160156 +util 96.750000 99.000000 94.000000 +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 153 : avg iteration time 46.33 seconds +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 153 : data loading time 0.00 +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 153 : avg forward pass time 12.24 +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 153 : backward pass time 10.14 +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 153 : optimizer step time 1.57 +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 153-------------------- +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229468 41.346142 41.176262 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.176758 67.285156 67.140625 +util 98.250000 100.000000 96.000000 +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 154 : avg iteration time 43.83 seconds +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 154 : data loading time 0.00 +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 154 : avg forward pass time 10.95 +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 154 : backward pass time 10.17 +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 154 : optimizer step time 1.60 +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 154-------------------- +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:02:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228927 41.343918 41.176247 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.628906 107.603516 +util 68.500000 71.000000 66.000000 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 155 : avg iteration time 159.01 seconds +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 155 : data loading time 0.00 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 155 : avg forward pass time 70.90 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 155 : backward pass time 8.42 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 155 : optimizer step time 0.39 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 155-------------------- +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6318 iter count: 1.0 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6318 iter count: 1.0 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6318 iter count: 1.0 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6318iter count: 1.0 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:05:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228927 41.343918 41.176247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.382812 67.335938 +util 98.250000 100.000000 96.000000 +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 156 : avg iteration time 50.39 seconds +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 156 : data loading time 0.00 +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 156 : avg forward pass time 14.24 +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 156 : backward pass time 10.15 +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 156 : optimizer step time 1.57 +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 156-------------------- +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:05:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229074 41.344536 41.176247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.313477 67.316406 67.304688 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 157 : avg iteration time 46.41 seconds +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 157 : data loading time 0.00 +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 157 : avg forward pass time 12.26 +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 157 : backward pass time 10.15 +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 157 : optimizer step time 1.57 +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 157-------------------- +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:06:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229402 41.345776 41.176247 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.323242 67.343750 67.316406 +util 91.500000 100.000000 88.000000 +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 158 : avg iteration time 46.67 seconds +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 158 : data loading time 0.00 +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 158 : avg forward pass time 12.40 +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 158 : backward pass time 10.14 +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 158 : optimizer step time 1.58 +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 158-------------------- +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:07:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229406 41.345795 41.176273 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.318359 67.324219 67.316406 +util 94.000000 100.000000 88.000000 +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 159 : avg iteration time 46.39 seconds +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 159 : data loading time 0.00 +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 159 : avg forward pass time 12.27 +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 159 : backward pass time 10.14 +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 159 : optimizer step time 1.56 +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 159-------------------- +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:08:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228978 41.343979 41.176273 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.707031 107.544922 +util 65.000000 67.000000 63.000000 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 160 : avg iteration time 164.52 seconds +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 160 : data loading time 0.00 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 160 : avg forward pass time 73.64 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 160 : backward pass time 8.41 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 160 : optimizer step time 0.39 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 160-------------------- +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6631 iter count: 1.0 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6631 iter count: 1.0 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6631 iter count: 1.0 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6631iter count: 1.0 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:11:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228992 41.343983 41.176403 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.329590 67.337891 67.304688 +util 95.500000 98.000000 93.000000 +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 161 : avg iteration time 44.80 seconds +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 161 : data loading time 0.00 +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 161 : avg forward pass time 11.46 +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 161 : backward pass time 10.14 +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 161 : optimizer step time 1.57 +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 161-------------------- +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:11:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229180 41.344711 41.176403 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.241699 67.363281 67.201172 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 162 : avg iteration time 46.44 seconds +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 162 : data loading time 0.00 +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 162 : avg forward pass time 12.28 +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 162 : backward pass time 10.14 +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 162 : optimizer step time 1.59 +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 162-------------------- +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:12:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229373 41.345444 41.176418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.212402 67.363281 67.162109 +util 96.250000 100.000000 87.000000 +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 163 : avg iteration time 46.23 seconds +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 163 : data loading time 0.00 +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 163 : avg forward pass time 12.20 +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 163 : backward pass time 10.13 +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 163 : optimizer step time 1.58 +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 163-------------------- +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:13:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229424 41.345695 41.176418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358887 67.363281 67.357422 +util 91.500000 94.000000 89.000000 +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 164 : avg iteration time 46.27 seconds +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 164 : data loading time 0.00 +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 164 : avg forward pass time 12.22 +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 164 : backward pass time 10.13 +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 164 : optimizer step time 1.57 +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 164-------------------- +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:14:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228989 41.343857 41.176414 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.687500 107.544922 +util 66.500000 71.000000 56.000000 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 165 : avg iteration time 161.94 seconds +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 165 : data loading time 0.00 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 165 : avg forward pass time 72.35 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 165 : backward pass time 8.44 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 165 : optimizer step time 0.39 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 165-------------------- +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6699 iter count: 1.0 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6699 iter count: 1.0 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6699 iter count: 1.0 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6699iter count: 1.0 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:16:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229011 41.343857 41.176449 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.254883 67.480469 67.179688 +util 97.000000 99.000000 94.000000 +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 166 : avg iteration time 50.14 seconds +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 166 : data loading time 0.00 +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 166 : avg forward pass time 14.14 +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 166 : backward pass time 10.13 +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 166 : optimizer step time 1.58 +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 166-------------------- +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:17:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229189 41.344597 41.176414 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.230469 67.500000 67.140625 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 167 : avg iteration time 49.03 seconds +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 167 : data loading time 0.00 +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 167 : avg forward pass time 13.60 +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 167 : backward pass time 10.13 +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 167 : optimizer step time 1.57 +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 167-------------------- +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:18:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229445 41.345596 41.176414 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.215820 67.441406 67.140625 +util 95.250000 98.000000 92.000000 +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 168 : avg iteration time 46.55 seconds +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 168 : data loading time 0.00 +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 168 : avg forward pass time 12.35 +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 168 : backward pass time 10.13 +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 168 : optimizer step time 1.56 +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 168-------------------- +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:19:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229445 41.345573 41.176434 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.210938 67.421875 67.140625 +util 93.750000 96.000000 91.000000 +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 169 : avg iteration time 46.39 seconds +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 169 : data loading time 0.00 +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 169 : avg forward pass time 12.27 +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 169 : backward pass time 10.14 +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 169 : optimizer step time 1.57 +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 169-------------------- +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:20:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228953 41.343605 41.176414 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.667969 107.623047 +util 68.250000 69.000000 66.000000 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 170 : avg iteration time 162.07 seconds +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 170 : data loading time 0.00 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 170 : avg forward pass time 72.39 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 170 : backward pass time 8.47 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 170 : optimizer step time 0.40 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 170-------------------- +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6650 iter count: 1.0 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6650 iter count: 1.0 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6650 iter count: 1.0 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6650iter count: 1.0 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:22:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228938 41.343559 41.176418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.245117 67.382812 67.199219 +util 96.000000 99.000000 87.000000 +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 171 : avg iteration time 47.62 seconds +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 171 : data loading time 0.00 +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 171 : avg forward pass time 12.87 +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 171 : backward pass time 10.14 +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 171 : optimizer step time 1.58 +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 171-------------------- +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:23:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229066 41.344036 41.176453 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.210938 67.363281 67.160156 +util 93.500000 99.000000 87.000000 +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 172 : avg iteration time 46.43 seconds +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 172 : data loading time 0.00 +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 172 : avg forward pass time 12.28 +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 172 : backward pass time 10.13 +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 172 : optimizer step time 1.57 +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 172-------------------- +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:24:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229321 41.345055 41.176453 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.215820 67.382812 67.160156 +util 96.750000 100.000000 89.000000 +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 173 : avg iteration time 46.57 seconds +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 173 : data loading time 0.00 +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 173 : avg forward pass time 12.35 +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 173 : backward pass time 10.15 +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 173 : optimizer step time 1.56 +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 173-------------------- +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:25:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229318 41.345032 41.176453 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.220703 67.402344 67.160156 +util 99.000000 100.000000 98.000000 +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 174 : avg iteration time 46.39 seconds +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 174 : data loading time 0.00 +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 174 : avg forward pass time 12.28 +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 174 : backward pass time 10.12 +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 174 : optimizer step time 1.56 +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 174-------------------- +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:25:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228876 41.343239 41.176491 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.687500 107.525391 +util 65.500000 70.000000 55.000000 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 175 : avg iteration time 159.42 seconds +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 175 : data loading time 0.00 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 175 : avg forward pass time 71.10 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 175 : backward pass time 8.42 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 175 : optimizer step time 0.40 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 175-------------------- +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6494 iter count: 1.0 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6494 iter count: 1.0 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6494 iter count: 1.0 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6494iter count: 1.0 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:28:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228882 41.343262 41.176491 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.303711 67.792969 67.140625 +util 92.750000 99.000000 90.000000 +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 176 : avg iteration time 47.66 seconds +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 176 : data loading time 0.00 +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 176 : avg forward pass time 12.90 +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 176 : backward pass time 10.14 +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 176 : optimizer step time 1.57 +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 176-------------------- +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:29:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229001 41.343731 41.176495 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.220703 67.460938 67.140625 +util 96.500000 99.000000 94.000000 +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 177 : avg iteration time 46.49 seconds +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 177 : data loading time 0.00 +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 177 : avg forward pass time 12.28 +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 177 : backward pass time 10.15 +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 177 : optimizer step time 1.58 +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 177-------------------- +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:30:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229363 41.345177 41.176495 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.245117 67.382812 67.199219 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 178 : avg iteration time 43.65 seconds +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 178 : data loading time 0.00 +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 178 : avg forward pass time 10.91 +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 178 : backward pass time 10.13 +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 178 : optimizer step time 1.57 +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 178-------------------- +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:30:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229362 41.345177 41.176495 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.206055 67.460938 67.121094 +util 93.250000 100.000000 90.000000 +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 179 : avg iteration time 49.27 seconds +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 179 : data loading time 0.00 +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 179 : avg forward pass time 13.70 +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 179 : backward pass time 10.14 +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 179 : optimizer step time 1.58 +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 179-------------------- +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:31:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229046 41.343880 41.176517 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.589844 107.564453 +util 63.500000 72.000000 55.000000 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 180 : avg iteration time 164.57 seconds +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 180 : data loading time 0.00 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 180 : avg forward pass time 73.67 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 180 : backward pass time 8.42 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 180 : optimizer step time 0.40 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 180-------------------- +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7051 iter count: 1.0 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7051 iter count: 1.0 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7051 iter count: 1.0 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7051iter count: 1.0 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:34:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229040 41.343880 41.176495 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.474609 67.773438 67.375000 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 181 : avg iteration time 47.59 seconds +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 181 : data loading time 0.00 +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 181 : avg forward pass time 12.85 +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 181 : backward pass time 10.17 +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 181 : optimizer step time 1.59 +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 181-------------------- +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:35:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229227 41.344627 41.176495 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.542969 67.570312 67.460938 +util 96.250000 100.000000 87.000000 +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 182 : avg iteration time 46.58 seconds +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 182 : data loading time 0.00 +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 182 : avg forward pass time 12.35 +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 182 : backward pass time 10.14 +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 182 : optimizer step time 1.57 +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 182-------------------- +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:35:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229697 41.346508 41.176495 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.713867 67.804688 67.441406 +util 92.250000 94.000000 90.000000 +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 183 : avg iteration time 49.08 seconds +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 183 : data loading time 0.00 +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 183 : avg forward pass time 13.59 +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 183 : backward pass time 10.16 +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 183 : optimizer step time 1.57 +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 183-------------------- +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:36:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229701 41.346508 41.176510 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.713867 67.804688 67.441406 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 184 : avg iteration time 43.68 seconds +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 184 : data loading time 0.00 +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 184 : avg forward pass time 10.90 +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 184 : backward pass time 10.15 +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 184 : optimizer step time 1.57 +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 184-------------------- +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:37:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229152 41.344288 41.176517 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.667969 107.544922 +util 66.250000 71.000000 55.000000 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 185 : avg iteration time 159.02 seconds +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 185 : data loading time 0.00 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 185 : avg forward pass time 70.88 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 185 : backward pass time 8.44 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 185 : optimizer step time 0.39 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 185-------------------- +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:40:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229175 41.344292 41.176495 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396484 67.695312 67.296875 +util 94.250000 96.000000 92.000000 +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 186 : avg iteration time 50.71 seconds +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 186 : data loading time 0.00 +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 186 : avg forward pass time 14.43 +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 186 : backward pass time 10.15 +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 186 : optimizer step time 1.57 +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 186-------------------- +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:40:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229239 41.344528 41.176517 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.406250 67.617188 67.335938 +util 90.750000 93.000000 88.000000 +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 187 : avg iteration time 43.75 seconds +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 187 : data loading time 0.00 +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 187 : avg forward pass time 10.93 +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 187 : backward pass time 10.15 +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 187 : optimizer step time 1.58 +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 187-------------------- +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:41:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229420 41.345257 41.176506 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.539062 67.335938 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 188 : avg iteration time 43.69 seconds +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 188 : data loading time 0.00 +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 188 : avg forward pass time 10.92 +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 188 : backward pass time 10.14 +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 188 : optimizer step time 1.56 +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 188-------------------- +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 16:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229478 41.345505 41.176495 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.391602 67.558594 67.335938 +util 97.500000 99.000000 95.000000 +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 189 : avg iteration time 48.99 seconds +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 189 : data loading time 0.00 +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 189 : avg forward pass time 13.56 +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 189 : backward pass time 10.17 +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 189 : optimizer step time 1.57 +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 189-------------------- +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229087 41.343914 41.176517 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.727051 107.863281 107.681641 +util 65.750000 70.000000 55.000000 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 190 : avg iteration time 164.84 seconds +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 190 : data loading time 0.00 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 190 : avg forward pass time 73.82 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 190 : backward pass time 8.42 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 190 : optimizer step time 0.40 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 190-------------------- +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6387 iter count: 1.0 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6387 iter count: 1.0 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6387 iter count: 1.0 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6387iter count: 1.0 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:45:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229086 41.343914 41.176517 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.427246 67.753906 67.318359 +util 93.250000 95.000000 91.000000 +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 191 : avg iteration time 50.30 seconds +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 191 : data loading time 0.00 +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 191 : avg forward pass time 14.21 +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 191 : backward pass time 10.16 +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 191 : optimizer step time 1.57 +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 191-------------------- +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:46:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229147 41.344162 41.176514 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.378418 67.441406 67.357422 +util 94.250000 96.000000 92.000000 +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 192 : avg iteration time 43.61 seconds +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 192 : data loading time 0.00 +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 192 : avg forward pass time 10.89 +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 192 : backward pass time 10.14 +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 192 : optimizer step time 1.58 +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 192-------------------- +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:47:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227808 41.344025 41.174889 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.383301 67.460938 67.357422 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 193 : avg iteration time 49.25 seconds +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 193 : data loading time 0.00 +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 193 : avg forward pass time 13.71 +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 193 : backward pass time 10.12 +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 193 : optimizer step time 1.55 +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 193-------------------- +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:48:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227791 41.344025 41.174843 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.378418 67.441406 67.357422 +util 91.250000 99.000000 88.000000 +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 194 : avg iteration time 49.27 seconds +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 194 : data loading time 0.00 +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 194 : avg forward pass time 13.69 +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 194 : backward pass time 10.16 +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 194 : optimizer step time 1.57 +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 194-------------------- +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:49:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227324 41.342125 41.174866 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.667969 107.583984 +util 65.500000 70.000000 54.000000 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 195 : avg iteration time 156.57 seconds +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 195 : data loading time 0.00 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 195 : avg forward pass time 69.65 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 195 : backward pass time 8.47 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 195 : optimizer step time 0.39 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 195-------------------- +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6064 iter count: 1.0 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6064 iter count: 1.0 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6064 iter count: 1.0 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6064iter count: 1.0 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:51:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227320 41.342129 41.174843 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.597656 67.414062 +util 95.750000 99.000000 86.000000 +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 196 : avg iteration time 50.40 seconds +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 196 : data loading time 0.00 +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 196 : avg forward pass time 14.26 +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 196 : backward pass time 10.15 +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 196 : optimizer step time 1.56 +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 196-------------------- +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:52:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227387 41.342373 41.174843 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.269531 67.597656 67.160156 +util 93.250000 95.000000 91.000000 +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 197 : avg iteration time 49.30 seconds +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 197 : data loading time 0.00 +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 197 : avg forward pass time 13.72 +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 197 : backward pass time 10.15 +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 197 : optimizer step time 1.57 +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 197-------------------- +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:53:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227637 41.343357 41.174843 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.391602 67.617188 67.316406 +util 93.750000 99.000000 88.000000 +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 198 : avg iteration time 46.73 seconds +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 198 : data loading time 0.00 +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 198 : avg forward pass time 12.42 +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 198 : backward pass time 10.16 +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 198 : optimizer step time 1.59 +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 198-------------------- +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:54:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227633 41.343357 41.174843 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.376953 67.558594 67.316406 +util 91.750000 99.000000 89.000000 +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 199 : avg iteration time 46.41 seconds +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 199 : data loading time 0.00 +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 199 : avg forward pass time 12.28 +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 199 : backward pass time 10.14 +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 199 : optimizer step time 1.58 +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 199-------------------- +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:55:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227235 41.341705 41.174850 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.628906 107.564453 +util 69.750000 71.000000 68.000000 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 200 : avg iteration time 164.35 seconds +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 200 : data loading time 0.00 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 200 : avg forward pass time 73.58 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 200 : backward pass time 8.41 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 200 : optimizer step time 0.39 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 200-------------------- +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6367 iter count: 1.0 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6367 iter count: 1.0 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6367 iter count: 1.0 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6367iter count: 1.0 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 16:57:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 16:57:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.35 seconds +[May 1, 2026 - 16:57:59 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 200 +[May 1, 2026 - 16:57:59 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000200 +[May 1, 2026 - 16:58:37 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 200 +[May 1, 2026 - 16:58:37 | INFO | fastgen.trainer:validate:358 ] Validation iteration 200 +[May 1, 2026 - 16:58:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:58:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.2s +[May 1, 2026 - 16:58:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.408 +[May 1, 2026 - 16:58:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:58:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:58:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.090 +[May 1, 2026 - 16:58:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:58:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:58:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.117 +[May 1, 2026 - 16:58:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:58:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:58:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.151 +[May 1, 2026 - 16:58:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:59:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:59:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.657 +[May 1, 2026 - 16:59:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:59:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:59:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.333 +[May 1, 2026 - 16:59:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:59:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:59:14 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.826 +[May 1, 2026 - 16:59:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:59:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:59:19 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.082 +[May 1, 2026 - 16:59:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:59:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:59:24 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.122 +[May 1, 2026 - 16:59:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 16:59:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 16:59:29 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.939 +[May 1, 2026 - 16:59:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 200-------------------- +[May 1, 2026 - 16:59:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 16:59:45 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 200 +[May 1, 2026 - 16:59:45 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.572 (n=10) +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228836 41.347324 41.175133 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.933594 68.085938 67.882812 +util 91.000000 93.000000 89.000000 +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 201 : avg iteration time 166.67 seconds +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 201 : data loading time 0.00 +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 201 : avg forward pass time 11.93 +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 201 : backward pass time 10.14 +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 201 : optimizer step time 1.56 +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 201-------------------- +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:00:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228836 41.347324 41.175133 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.948242 68.203125 67.863281 +util 93.250000 95.000000 91.000000 +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 202 : avg iteration time 49.23 seconds +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 202 : data loading time 0.00 +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 202 : avg forward pass time 13.65 +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 202 : backward pass time 10.18 +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 202 : optimizer step time 1.58 +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 202-------------------- +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:01:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228858 41.347324 41.175148 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.943359 68.066406 67.902344 +util 93.750000 99.000000 88.000000 +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 203 : avg iteration time 46.53 seconds +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 203 : data loading time 0.00 +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 203 : avg forward pass time 12.33 +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 203 : backward pass time 10.15 +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 203 : optimizer step time 1.57 +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 203-------------------- +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228882 41.347324 41.175133 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.958008 68.066406 67.921875 +util 94.250000 96.000000 92.000000 +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 204 : avg iteration time 43.90 seconds +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 204 : data loading time 0.00 +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 204 : avg forward pass time 11.00 +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 204 : backward pass time 10.15 +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 204 : optimizer step time 1.59 +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 204-------------------- +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:02:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227874 41.343246 41.175133 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.589844 107.564453 +util 67.500000 69.000000 66.000000 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 205 : avg iteration time 161.85 seconds +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 205 : data loading time 0.00 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 205 : avg forward pass time 72.32 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 205 : backward pass time 8.41 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 205 : optimizer step time 0.40 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 205-------------------- +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6162 iter count: 1.0 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6162 iter count: 1.0 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6162 iter count: 1.0 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6162iter count: 1.0 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:05:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227859 41.343243 41.175133 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.367188 67.460938 67.335938 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 206 : avg iteration time 47.65 seconds +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 206 : data loading time 0.00 +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 206 : avg forward pass time 12.88 +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 206 : backward pass time 10.14 +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 206 : optimizer step time 1.58 +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 206-------------------- +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228054 41.343994 41.175133 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.420898 67.558594 67.375000 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 207 : avg iteration time 46.39 seconds +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 207 : data loading time 0.00 +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 207 : avg forward pass time 12.27 +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 207 : backward pass time 10.13 +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 207 : optimizer step time 1.57 +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 207-------------------- +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:07:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228144 41.344246 41.175224 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.542969 67.578125 67.531250 +util 92.000000 100.000000 89.000000 +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 208 : avg iteration time 46.63 seconds +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 208 : data loading time 0.00 +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 208 : avg forward pass time 12.37 +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 208 : backward pass time 10.16 +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 208 : optimizer step time 1.57 +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 208-------------------- +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:07:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228259 41.344730 41.175224 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.191406 67.519531 67.082031 +util 97.750000 99.000000 96.000000 +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 209 : avg iteration time 49.11 seconds +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 209 : data loading time 0.00 +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 209 : avg forward pass time 13.63 +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 209 : backward pass time 10.14 +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 209 : optimizer step time 1.57 +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 209-------------------- +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:08:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227736 41.342636 41.175228 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.254395 108.443359 107.687500 +util 64.500000 65.000000 63.000000 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 210 : avg iteration time 159.18 seconds +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 210 : data loading time 0.00 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 210 : avg forward pass time 70.94 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 210 : backward pass time 8.49 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 210 : optimizer step time 0.39 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 210-------------------- +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6299 iter count: 1.0 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6299 iter count: 1.0 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6299 iter count: 1.0 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6299iter count: 1.0 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:11:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227792 41.342850 41.175228 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.382812 67.355469 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 211 : avg iteration time 47.66 seconds +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 211 : data loading time 0.00 +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 211 : avg forward pass time 12.91 +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 211 : backward pass time 10.15 +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 211 : optimizer step time 1.56 +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 211-------------------- +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:12:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228040 41.343849 41.175228 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.636719 67.335938 +util 93.000000 99.000000 86.000000 +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 212 : avg iteration time 49.23 seconds +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 212 : data loading time 0.00 +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 212 : avg forward pass time 13.70 +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 212 : backward pass time 10.12 +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 212 : optimizer step time 1.56 +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 212-------------------- +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:12:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228360 41.345047 41.175228 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.539062 67.335938 +util 90.250000 92.000000 88.000000 +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 213 : avg iteration time 43.65 seconds +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 213 : data loading time 0.00 +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 213 : avg forward pass time 10.88 +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 213 : backward pass time 10.15 +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 213 : optimizer step time 1.59 +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 213-------------------- +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:13:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228366 41.345047 41.175228 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.445312 67.773438 67.335938 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 214 : avg iteration time 49.31 seconds +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 214 : data loading time 0.00 +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 214 : avg forward pass time 13.73 +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 214 : backward pass time 10.14 +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 214 : optimizer step time 1.57 +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 214-------------------- +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:14:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227886 41.343063 41.175236 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.687500 107.564453 +util 61.750000 73.000000 57.000000 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 215 : avg iteration time 162.07 seconds +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 215 : data loading time 0.00 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 215 : avg forward pass time 72.43 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 215 : backward pass time 8.41 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 215 : optimizer step time 0.40 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 215-------------------- +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6670 iter count: 1.0 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6670 iter count: 1.0 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6670 iter count: 1.0 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6670iter count: 1.0 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:17:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227875 41.343067 41.175255 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.914062 67.921875 67.890625 +util 96.250000 98.000000 94.000000 +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 216 : avg iteration time 47.68 seconds +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 216 : data loading time 0.00 +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 216 : avg forward pass time 12.91 +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 216 : backward pass time 10.15 +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 216 : optimizer step time 1.57 +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 216-------------------- +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:18:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228074 41.343792 41.175323 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.538086 67.968750 67.394531 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 217 : avg iteration time 46.44 seconds +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 217 : data loading time 0.00 +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 217 : avg forward pass time 12.27 +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 217 : backward pass time 10.14 +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 217 : optimizer step time 1.57 +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 217-------------------- +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:18:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228365 41.344959 41.175323 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.923828 67.929688 67.921875 +util 91.500000 93.000000 89.000000 +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 218 : avg iteration time 49.10 seconds +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 218 : data loading time 0.00 +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 218 : avg forward pass time 13.61 +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 218 : backward pass time 10.15 +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 218 : optimizer step time 1.56 +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 218-------------------- +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:19:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228445 41.345230 41.175373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.865234 67.902344 67.753906 +util 98.250000 100.000000 96.000000 +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 219 : avg iteration time 46.49 seconds +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 219 : data loading time 0.00 +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 219 : avg forward pass time 12.31 +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 219 : backward pass time 10.14 +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 219 : optimizer step time 1.58 +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 219-------------------- +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:20:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227917 41.343098 41.175373 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.648438 107.642578 +util 67.500000 69.000000 66.000000 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 220 : avg iteration time 162.01 seconds +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 220 : data loading time 0.00 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 220 : avg forward pass time 72.39 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 220 : backward pass time 8.41 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 220 : optimizer step time 0.40 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 220-------------------- +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6289 iter count: 1.0 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6289 iter count: 1.0 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6289 iter count: 1.0 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6289iter count: 1.0 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:23:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227911 41.343075 41.175373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.500488 67.812500 67.396484 +util 91.500000 100.000000 88.000000 +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 221 : avg iteration time 44.86 seconds +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 221 : data loading time 0.00 +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 221 : avg forward pass time 11.50 +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 221 : backward pass time 10.15 +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 221 : optimizer step time 1.56 +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 221-------------------- +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:23:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228042 41.343597 41.175373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.866699 67.871094 67.865234 +util 95.250000 97.000000 93.000000 +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 222 : avg iteration time 49.09 seconds +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 222 : data loading time 0.00 +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 222 : avg forward pass time 13.62 +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 222 : backward pass time 10.15 +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 222 : optimizer step time 1.57 +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 222-------------------- +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:24:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228419 41.345085 41.175396 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.876465 67.910156 67.865234 +util 97.750000 99.000000 96.000000 +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 223 : avg iteration time 49.23 seconds +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 223 : data loading time 0.00 +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 223 : avg forward pass time 13.67 +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 223 : backward pass time 10.16 +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 223 : optimizer step time 1.57 +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 223-------------------- +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228476 41.345314 41.175373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.832520 67.865234 67.734375 +util 92.000000 93.000000 90.000000 +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 224 : avg iteration time 43.74 seconds +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 224 : data loading time 0.00 +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 224 : avg forward pass time 10.95 +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 224 : backward pass time 10.14 +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 224 : optimizer step time 1.57 +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 224-------------------- +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227944 41.343201 41.175373 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.628906 107.525391 +util 67.000000 71.000000 56.000000 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 225 : avg iteration time 167.13 seconds +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 225 : data loading time 0.00 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 225 : avg forward pass time 74.96 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 225 : backward pass time 8.40 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 225 : optimizer step time 0.39 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 225-------------------- +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6230 iter count: 1.0 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6230 iter count: 1.0 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6230 iter count: 1.0 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6230iter count: 1.0 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:28:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227944 41.343201 41.175373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.533203 67.949219 67.394531 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 226 : avg iteration time 50.16 seconds +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 226 : data loading time 0.00 +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 226 : avg forward pass time 14.14 +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 226 : backward pass time 10.15 +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 226 : optimizer step time 1.57 +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 226-------------------- +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:29:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228127 41.343914 41.175373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.421875 67.375000 +util 92.000000 93.000000 90.000000 +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 227 : avg iteration time 49.12 seconds +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 227 : data loading time 0.00 +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 227 : avg forward pass time 13.63 +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 227 : backward pass time 10.13 +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 227 : optimizer step time 1.57 +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 227-------------------- +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:30:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228332 41.344666 41.175453 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.636719 67.355469 +util 95.000000 96.000000 93.000000 +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 228 : avg iteration time 43.64 seconds +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 228 : data loading time 0.00 +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 228 : avg forward pass time 10.91 +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 228 : backward pass time 10.13 +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 228 : optimizer step time 1.57 +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 228-------------------- +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:31:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228418 41.344963 41.175503 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.391602 67.500000 67.355469 +util 98.000000 99.000000 96.000000 +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 229 : avg iteration time 49.23 seconds +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 229 : data loading time 0.00 +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 229 : avg forward pass time 13.69 +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 229 : backward pass time 10.14 +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 229 : optimizer step time 1.58 +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 229-------------------- +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:32:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228250 41.344254 41.175533 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.667969 107.623047 +util 67.000000 68.000000 65.000000 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 230 : avg iteration time 159.61 seconds +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 230 : data loading time 0.00 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 230 : avg forward pass time 71.20 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 230 : backward pass time 8.44 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 230 : optimizer step time 0.39 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 230-------------------- +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:34:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227992 41.343246 41.175510 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.349121 67.910156 67.162109 +util 95.750000 97.000000 94.000000 +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 231 : avg iteration time 50.57 seconds +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 231 : data loading time 0.00 +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 231 : avg forward pass time 14.33 +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 231 : backward pass time 10.17 +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 231 : optimizer step time 1.56 +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 231-------------------- +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:35:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227992 41.343246 41.175510 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.236816 67.753906 67.064453 +util 93.250000 100.000000 87.000000 +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 232 : avg iteration time 46.89 seconds +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 232 : data loading time 0.00 +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 232 : avg forward pass time 12.49 +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 232 : backward pass time 10.15 +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 232 : optimizer step time 1.57 +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 232-------------------- +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:36:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228131 41.343742 41.175533 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.251465 67.753906 67.083984 +util 89.500000 90.000000 89.000000 +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 233 : avg iteration time 43.66 seconds +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 233 : data loading time 0.00 +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 233 : avg forward pass time 10.88 +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 233 : backward pass time 10.14 +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 233 : optimizer step time 1.61 +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 233-------------------- +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:37:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228124 41.343750 41.175510 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.295410 67.929688 67.083984 +util 91.000000 92.000000 89.000000 +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 234 : avg iteration time 46.59 seconds +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 234 : data loading time 0.00 +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 234 : avg forward pass time 12.37 +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 234 : backward pass time 10.15 +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 234 : optimizer step time 1.56 +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 234-------------------- +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:37:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228122 41.343742 41.175510 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.678223 107.785156 107.642578 +util 62.500000 64.000000 61.000000 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 235 : avg iteration time 159.03 seconds +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 235 : data loading time 0.00 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 235 : avg forward pass time 70.90 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 235 : backward pass time 8.43 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 235 : optimizer step time 0.39 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 235-------------------- +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6172 iter count: 1.0 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6172 iter count: 1.0 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6172 iter count: 1.0 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6172iter count: 1.0 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:40:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228083 41.343586 41.175510 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.890625 67.337891 +util 91.750000 99.000000 89.000000 +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 236 : avg iteration time 44.74 seconds +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 236 : data loading time 0.00 +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 236 : avg forward pass time 11.45 +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 236 : backward pass time 10.12 +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 236 : optimizer step time 1.56 +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 236-------------------- +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:41:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228089 41.343586 41.175533 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373535 68.007812 67.162109 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 237 : avg iteration time 43.72 seconds +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 237 : data loading time 0.00 +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 237 : avg forward pass time 10.95 +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 237 : backward pass time 10.14 +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 237 : optimizer step time 1.56 +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 237-------------------- +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:42:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228090 41.343586 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.851562 67.318359 +util 93.000000 100.000000 86.000000 +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 238 : avg iteration time 49.06 seconds +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 238 : data loading time 0.00 +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 238 : avg forward pass time 13.61 +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 238 : backward pass time 10.13 +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 238 : optimizer step time 1.57 +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 238-------------------- +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:42:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228083 41.343586 41.175510 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.500488 67.988281 67.337891 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 239 : avg iteration time 49.17 seconds +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 239 : data loading time 0.00 +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 239 : avg forward pass time 13.62 +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 239 : backward pass time 10.16 +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 239 : optimizer step time 1.58 +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 239-------------------- +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:43:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228215 41.344112 41.175514 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.673340 107.707031 107.662109 +util 70.500000 72.000000 69.000000 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 240 : avg iteration time 161.62 seconds +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 240 : data loading time 0.00 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 240 : avg forward pass time 72.21 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 240 : backward pass time 8.43 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 240 : optimizer step time 0.39 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 240-------------------- +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6602 iter count: 1.0 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6602 iter count: 1.0 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6602 iter count: 1.0 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6602iter count: 1.0 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:46:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228129 41.343746 41.175537 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.139160 67.480469 67.025391 +util 91.250000 100.000000 88.000000 +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 241 : avg iteration time 47.64 seconds +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 241 : data loading time 0.00 +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 241 : avg forward pass time 12.88 +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 241 : backward pass time 10.13 +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 241 : optimizer step time 1.59 +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 241-------------------- +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:47:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:47:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228133 41.343746 41.175552 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.183105 67.597656 67.044922 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 17:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 242 : avg iteration time 43.73 seconds +[May 1, 2026 - 17:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 242 : data loading time 0.00 +[May 1, 2026 - 17:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 242 : avg forward pass time 10.95 +[May 1, 2026 - 17:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 242 : backward pass time 10.14 +[May 1, 2026 - 17:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 242 : optimizer step time 1.56 +[May 1, 2026 - 17:47:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 242-------------------- +[May 1, 2026 - 17:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 17:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 17:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:47:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228133 41.343746 41.175514 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.134277 67.402344 67.044922 +util 92.000000 100.000000 89.000000 +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 243 : avg iteration time 46.47 seconds +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 243 : data loading time 0.00 +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 243 : avg forward pass time 12.31 +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 243 : backward pass time 10.14 +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 243 : optimizer step time 1.57 +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 243-------------------- +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:48:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228253 41.344238 41.175514 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.158691 67.500000 67.044922 +util 92.750000 99.000000 86.000000 +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 244 : avg iteration time 43.77 seconds +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 244 : data loading time 0.00 +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 244 : avg forward pass time 10.95 +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 244 : backward pass time 10.14 +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 244 : optimizer step time 1.58 +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 244-------------------- +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:49:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228151 41.343796 41.175541 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.667969 107.544922 +util 59.750000 61.000000 58.000000 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 245 : avg iteration time 159.26 seconds +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 245 : data loading time 0.00 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 245 : avg forward pass time 71.02 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 245 : backward pass time 8.42 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 245 : optimizer step time 0.39 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 245-------------------- +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6377 iter count: 1.0 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6377 iter count: 1.0 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6377 iter count: 1.0 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6377iter count: 1.0 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:52:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228146 41.343803 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.083984 67.441406 66.964844 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 246 : avg iteration time 47.53 seconds +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 246 : data loading time 0.00 +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 246 : avg forward pass time 12.85 +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 246 : backward pass time 10.14 +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 246 : optimizer step time 1.57 +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 246-------------------- +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:52:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228148 41.343815 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.220703 67.460938 67.140625 +util 96.750000 100.000000 89.000000 +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 247 : avg iteration time 46.40 seconds +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 247 : data loading time 0.00 +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 247 : avg forward pass time 12.28 +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 247 : backward pass time 10.13 +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 247 : optimizer step time 1.57 +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 247-------------------- +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:53:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228152 41.343815 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.235352 67.460938 67.160156 +util 97.000000 98.000000 95.000000 +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 248 : avg iteration time 49.14 seconds +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 248 : data loading time 0.00 +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 248 : avg forward pass time 13.62 +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 248 : backward pass time 10.15 +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 248 : optimizer step time 1.59 +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 248-------------------- +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:54:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228275 41.344299 41.175541 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.235352 67.460938 67.160156 +util 89.500000 90.000000 88.000000 +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 249 : avg iteration time 46.31 seconds +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 249 : data loading time 0.00 +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 249 : avg forward pass time 12.23 +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 249 : backward pass time 10.15 +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 249 : optimizer step time 1.57 +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 249-------------------- +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:55:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227993 41.343159 41.175529 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.546387 107.667969 107.505859 +util 62.500000 63.000000 61.000000 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 250 : avg iteration time 167.25 seconds +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 250 : data loading time 0.00 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 250 : avg forward pass time 75.01 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 250 : backward pass time 8.41 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 250 : optimizer step time 0.39 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 250-------------------- +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6113 iter count: 1.0 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6113 iter count: 1.0 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6113 iter count: 1.0 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6113iter count: 1.0 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:58:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228167 41.343884 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.266113 67.812500 67.083984 +util 91.250000 92.000000 89.000000 +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 251 : avg iteration time 50.17 seconds +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 251 : data loading time 0.00 +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 251 : avg forward pass time 14.17 +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 251 : backward pass time 10.14 +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 251 : optimizer step time 1.57 +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 251-------------------- +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:58:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228230 41.344135 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.202637 67.500000 67.103516 +util 99.000000 100.000000 97.000000 +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 252 : avg iteration time 49.09 seconds +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 252 : data loading time 0.00 +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 252 : avg forward pass time 13.61 +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 252 : backward pass time 10.15 +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 252 : optimizer step time 1.57 +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 252-------------------- +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 17:59:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228236 41.344135 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.163574 67.402344 67.083984 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 253 : avg iteration time 46.46 seconds +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 253 : data loading time 0.00 +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 253 : avg forward pass time 12.30 +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 253 : backward pass time 10.14 +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 253 : optimizer step time 1.56 +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 253-------------------- +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:00:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228363 41.344620 41.175541 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.158691 67.500000 67.044922 +util 96.000000 97.000000 94.000000 +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 254 : avg iteration time 46.51 seconds +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 254 : data loading time 0.00 +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 254 : avg forward pass time 12.32 +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 254 : backward pass time 10.13 +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 254 : optimizer step time 1.59 +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 254-------------------- +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:01:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228128 41.343586 41.175518 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.589844 107.564453 +util 66.750000 68.000000 66.000000 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 255 : avg iteration time 161.83 seconds +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 255 : data loading time 0.00 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 255 : avg forward pass time 72.31 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 255 : backward pass time 8.44 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 255 : optimizer step time 0.40 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 255-------------------- +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6582 iter count: 1.0 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6582 iter count: 1.0 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6582 iter count: 1.0 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6582iter count: 1.0 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:03:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228128 41.343586 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.792969 67.376953 +util 94.250000 95.000000 92.000000 +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 256 : avg iteration time 47.77 seconds +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 256 : data loading time 0.00 +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 256 : avg forward pass time 12.97 +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 256 : backward pass time 10.13 +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 256 : optimizer step time 1.59 +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 256-------------------- +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:04:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228193 41.343838 41.175522 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.890625 67.337891 +util 91.750000 93.000000 90.000000 +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 257 : avg iteration time 43.75 seconds +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 257 : data loading time 0.00 +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 257 : avg forward pass time 10.95 +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 257 : backward pass time 10.14 +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 257 : optimizer step time 1.56 +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 257-------------------- +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:05:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228192 41.343838 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358887 67.421875 67.337891 +util 96.000000 97.000000 95.000000 +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 258 : avg iteration time 49.05 seconds +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 258 : data loading time 0.00 +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 258 : avg forward pass time 13.59 +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 258 : backward pass time 10.16 +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 258 : optimizer step time 1.58 +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 258-------------------- +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:06:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228194 41.343838 41.175518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373535 67.539062 67.318359 +util 92.500000 93.000000 91.000000 +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 259 : avg iteration time 46.36 seconds +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 259 : data loading time 0.00 +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 259 : avg forward pass time 12.24 +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 259 : backward pass time 10.16 +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 259 : optimizer step time 1.57 +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 259-------------------- +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:07:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228093 41.343403 41.175549 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.707031 107.603516 +util 68.750000 69.000000 68.000000 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 260 : avg iteration time 159.09 seconds +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 260 : data loading time 0.00 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 260 : avg forward pass time 70.91 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 260 : backward pass time 8.44 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 260 : optimizer step time 0.39 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 260-------------------- +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6279 iter count: 1.0 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6279 iter count: 1.0 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6279 iter count: 1.0 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6279iter count: 1.0 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:09:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228088 41.343403 41.175529 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.095215 67.480469 66.966797 +util 96.750000 98.000000 95.000000 +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 261 : avg iteration time 44.96 seconds +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 261 : data loading time 0.00 +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 261 : avg forward pass time 11.56 +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 261 : backward pass time 10.12 +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 261 : optimizer step time 1.58 +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 261-------------------- +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:10:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228083 41.343380 41.175529 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.270996 67.480469 67.201172 +util 96.750000 98.000000 95.000000 +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 262 : avg iteration time 49.00 seconds +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 262 : data loading time 0.00 +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 262 : avg forward pass time 13.58 +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 262 : backward pass time 10.14 +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 262 : optimizer step time 1.57 +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 262-------------------- +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:11:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228091 41.343391 41.175529 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.129395 67.500000 67.005859 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 263 : avg iteration time 43.64 seconds +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 263 : data loading time 0.00 +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 263 : avg forward pass time 10.89 +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 263 : backward pass time 10.14 +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 263 : optimizer step time 1.58 +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 263-------------------- +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:12:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228117 41.343395 41.175529 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.432129 67.480469 67.416016 +util 93.250000 100.000000 87.000000 +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 264 : avg iteration time 46.44 seconds +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 264 : data loading time 0.00 +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 264 : avg forward pass time 12.30 +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 264 : backward pass time 10.14 +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 264 : optimizer step time 1.56 +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 264-------------------- +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:12:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228026 41.342999 41.175529 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.648438 107.583984 +util 62.500000 64.000000 61.000000 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 265 : avg iteration time 159.08 seconds +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 265 : data loading time 0.00 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 265 : avg forward pass time 70.93 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 265 : backward pass time 8.42 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 265 : optimizer step time 0.39 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 265-------------------- +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6074 iter count: 1.0 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6074 iter count: 1.0 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6074 iter count: 1.0 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6074iter count: 1.0 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:15:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228030 41.342999 41.175579 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.206055 67.636719 67.062500 +util 89.250000 90.000000 87.000000 +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 266 : avg iteration time 47.55 seconds +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 266 : data loading time 0.00 +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 266 : avg forward pass time 12.86 +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 266 : backward pass time 10.14 +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 266 : optimizer step time 1.58 +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 266-------------------- +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:16:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228121 41.343273 41.175579 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.949219 67.218750 +util 91.500000 92.000000 90.000000 +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 267 : avg iteration time 46.42 seconds +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 267 : data loading time 0.00 +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 267 : avg forward pass time 12.29 +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 267 : backward pass time 10.13 +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 267 : optimizer step time 1.58 +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 267-------------------- +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:17:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228168 41.343502 41.175579 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.636719 67.355469 +util 96.500000 97.000000 95.000000 +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 268 : avg iteration time 49.05 seconds +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 268 : data loading time 0.00 +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 268 : avg forward pass time 13.60 +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 268 : backward pass time 10.13 +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 268 : optimizer step time 1.57 +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 268-------------------- +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:17:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228189 41.343559 41.175579 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499023 67.871094 67.375000 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 269 : avg iteration time 43.84 seconds +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 269 : data loading time 0.00 +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 269 : avg forward pass time 10.98 +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 269 : backward pass time 10.13 +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 269 : optimizer step time 1.59 +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 269-------------------- +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227936 41.342495 41.175587 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.746094 107.564453 +util 70.000000 71.000000 69.000000 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 270 : avg iteration time 158.85 seconds +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 270 : data loading time 0.00 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 270 : avg forward pass time 70.83 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 270 : backward pass time 8.42 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 270 : optimizer step time 0.40 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 270-------------------- +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6309 iter count: 1.0 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6309 iter count: 1.0 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6309 iter count: 1.0 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6309iter count: 1.0 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:21:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227931 41.342495 41.175587 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.968750 67.140625 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 271 : avg iteration time 50.21 seconds +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 271 : data loading time 0.00 +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 271 : avg forward pass time 14.18 +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 271 : backward pass time 10.12 +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 271 : optimizer step time 1.58 +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 271-------------------- +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:22:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228048 41.342987 41.175602 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.542969 67.988281 67.394531 +util 90.500000 91.000000 89.000000 +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 272 : avg iteration time 43.67 seconds +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 272 : data loading time 0.00 +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 272 : avg forward pass time 10.89 +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 272 : backward pass time 10.14 +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 272 : optimizer step time 1.59 +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 272-------------------- +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228371 41.344231 41.175606 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.528320 67.988281 67.375000 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 273 : avg iteration time 49.49 seconds +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 273 : data loading time 0.00 +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 273 : avg forward pass time 13.80 +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 273 : backward pass time 10.16 +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 273 : optimizer step time 1.58 +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 273-------------------- +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228369 41.344231 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.567383 67.968750 67.433594 +util 90.500000 91.000000 89.000000 +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 274 : avg iteration time 43.71 seconds +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 274 : data loading time 0.00 +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 274 : avg forward pass time 10.91 +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 274 : backward pass time 10.15 +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 274 : optimizer step time 1.59 +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 274-------------------- +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:24:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227942 41.342476 41.175621 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.663574 107.667969 107.662109 +util 65.750000 72.000000 61.000000 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 275 : avg iteration time 161.75 seconds +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 275 : data loading time 0.00 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 275 : avg forward pass time 72.27 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 275 : backward pass time 8.43 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 275 : optimizer step time 0.40 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 275-------------------- +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6318 iter count: 1.0 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6318 iter count: 1.0 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6318 iter count: 1.0 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6318iter count: 1.0 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:27:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227936 41.342476 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.376953 67.441406 67.355469 +util 93.500000 94.000000 92.000000 +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 276 : avg iteration time 50.29 seconds +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 276 : data loading time 0.00 +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 276 : avg forward pass time 14.21 +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 276 : backward pass time 10.14 +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 276 : optimizer step time 1.57 +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 276-------------------- +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:27:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228074 41.343006 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.357422 67.480469 67.316406 +util 93.000000 99.000000 87.000000 +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 277 : avg iteration time 49.32 seconds +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 277 : data loading time 0.00 +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 277 : avg forward pass time 13.74 +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 277 : backward pass time 10.13 +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 277 : optimizer step time 1.58 +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 277-------------------- +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:28:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228370 41.344212 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.519531 67.433594 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 278 : avg iteration time 49.17 seconds +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 278 : data loading time 0.00 +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 278 : avg forward pass time 13.65 +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 278 : backward pass time 10.14 +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 278 : optimizer step time 1.57 +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 278-------------------- +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:29:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228381 41.344219 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.391602 67.500000 67.355469 +util 97.500000 98.000000 96.000000 +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 279 : avg iteration time 43.76 seconds +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 279 : data loading time 0.00 +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 279 : avg forward pass time 10.94 +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 279 : backward pass time 10.14 +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 279 : optimizer step time 1.58 +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 279-------------------- +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:30:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227967 41.342579 41.175621 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.628906 107.583984 +util 65.250000 66.000000 64.000000 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 280 : avg iteration time 164.40 seconds +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 280 : data loading time 0.00 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 280 : avg forward pass time 73.56 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 280 : backward pass time 8.47 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 280 : optimizer step time 0.40 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 280-------------------- +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6230 iter count: 1.0 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6230 iter count: 1.0 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6230 iter count: 1.0 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6230iter count: 1.0 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:32:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227967 41.342579 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.832031 67.357422 +util 90.500000 91.000000 89.000000 +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 281 : avg iteration time 44.73 seconds +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 281 : data loading time 0.00 +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 281 : avg forward pass time 11.45 +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 281 : backward pass time 10.13 +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 281 : optimizer step time 1.56 +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 281-------------------- +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:33:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228089 41.343082 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.393066 67.441406 67.376953 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 282 : avg iteration time 49.13 seconds +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 282 : data loading time 0.00 +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 282 : avg forward pass time 13.65 +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 282 : backward pass time 10.14 +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 282 : optimizer step time 1.56 +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 282-------------------- +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:34:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228541 41.344887 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.422363 67.500000 67.396484 +util 90.500000 91.000000 89.000000 +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 283 : avg iteration time 49.27 seconds +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 283 : data loading time 0.00 +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 283 : avg forward pass time 13.70 +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 283 : backward pass time 10.14 +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 283 : optimizer step time 1.58 +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 283-------------------- +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:35:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228653 41.345345 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.500000 67.474609 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 284 : avg iteration time 46.70 seconds +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 284 : data loading time 0.00 +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 284 : avg forward pass time 12.41 +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 284 : backward pass time 10.15 +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 284 : optimizer step time 1.57 +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 284-------------------- +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:36:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228096 41.343098 41.175621 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.103027 108.248047 107.667969 +util 66.000000 67.000000 65.000000 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 285 : avg iteration time 161.58 seconds +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 285 : data loading time 0.00 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 285 : avg forward pass time 72.18 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 285 : backward pass time 8.42 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 285 : optimizer step time 0.39 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 285-------------------- +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6562 iter count: 1.0 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6562 iter count: 1.0 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6562 iter count: 1.0 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6562iter count: 1.0 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:38:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228102 41.343121 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.929688 67.238281 +util 95.500000 96.000000 94.000000 +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 286 : avg iteration time 44.97 seconds +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 286 : data loading time 0.00 +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 286 : avg forward pass time 11.58 +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 286 : backward pass time 10.12 +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 286 : optimizer step time 1.57 +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 286-------------------- +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:39:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228165 41.343369 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.469727 67.812500 67.355469 +util 91.500000 92.000000 90.000000 +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 287 : avg iteration time 48.96 seconds +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 287 : data loading time 0.00 +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 287 : avg forward pass time 13.54 +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 287 : backward pass time 10.16 +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 287 : optimizer step time 1.58 +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 287-------------------- +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:40:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228410 41.344349 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 67.910156 67.355469 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 288 : avg iteration time 49.11 seconds +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 288 : data loading time 0.00 +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 288 : avg forward pass time 13.62 +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 288 : backward pass time 10.14 +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 288 : optimizer step time 1.58 +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 288-------------------- +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:41:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228416 41.344353 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.430664 67.773438 67.316406 +util 96.500000 97.000000 95.000000 +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 289 : avg iteration time 49.15 seconds +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 289 : data loading time 0.00 +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 289 : avg forward pass time 13.68 +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 289 : backward pass time 10.11 +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 289 : optimizer step time 1.55 +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 289-------------------- +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:42:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228008 41.342743 41.175621 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.667969 107.583984 +util 65.750000 66.000000 65.000000 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 290 : avg iteration time 164.40 seconds +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 290 : data loading time 0.00 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 290 : avg forward pass time 73.61 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 290 : backward pass time 8.41 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 290 : optimizer step time 0.39 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 290-------------------- +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6035 iter count: 1.0 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6035 iter count: 1.0 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6035 iter count: 1.0 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6035iter count: 1.0 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:44:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228018 41.342731 41.175659 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.807129 67.845703 67.695312 +util 96.500000 97.000000 95.000000 +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 291 : avg iteration time 44.80 seconds +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 291 : data loading time 0.00 +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 291 : avg forward pass time 11.47 +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 291 : backward pass time 10.14 +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 291 : optimizer step time 1.58 +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 291-------------------- +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228125 41.343208 41.175621 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.929199 67.949219 67.921875 +util 91.500000 92.000000 90.000000 +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 292 : avg iteration time 49.14 seconds +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 292 : data loading time 0.00 +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 292 : avg forward pass time 13.64 +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 292 : backward pass time 10.16 +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 292 : optimizer step time 1.57 +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 292-------------------- +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:46:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228380 41.344212 41.175625 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729004 67.792969 67.707031 +util 92.750000 93.000000 92.000000 +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 293 : avg iteration time 46.41 seconds +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 293 : data loading time 0.00 +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 293 : avg forward pass time 12.27 +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 293 : backward pass time 10.16 +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 293 : optimizer step time 1.57 +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 293-------------------- +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:47:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228451 41.344482 41.175636 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.807129 68.046875 67.726562 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 294 : avg iteration time 46.49 seconds +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 294 : data loading time 0.00 +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 294 : avg forward pass time 12.31 +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 294 : backward pass time 10.15 +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 294 : optimizer step time 1.58 +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 294-------------------- +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:47:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227970 41.342583 41.175625 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.751465 107.779297 107.667969 +util 65.250000 67.000000 64.000000 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 295 : avg iteration time 164.37 seconds +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 295 : data loading time 0.00 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 295 : avg forward pass time 73.60 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 295 : backward pass time 8.41 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 295 : optimizer step time 0.40 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 295-------------------- +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5928 iter count: 1.0 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5928 iter count: 1.0 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5928 iter count: 1.0 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5928iter count: 1.0 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:50:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227983 41.342583 41.175629 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.279785 67.929688 67.062500 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 296 : avg iteration time 44.86 seconds +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 296 : data loading time 0.00 +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 296 : avg forward pass time 11.49 +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 296 : backward pass time 10.14 +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 296 : optimizer step time 1.58 +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 296-------------------- +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:51:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228107 41.343082 41.175629 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.338379 67.929688 67.140625 +util 92.500000 93.000000 91.000000 +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 297 : avg iteration time 43.69 seconds +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 297 : data loading time 0.00 +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 297 : avg forward pass time 10.91 +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 297 : backward pass time 10.13 +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 297 : optimizer step time 1.56 +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 297-------------------- +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:52:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228362 41.344101 41.175629 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411621 67.929688 67.238281 +util 94.500000 95.000000 93.000000 +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 298 : avg iteration time 49.02 seconds +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 298 : data loading time 0.00 +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 298 : avg forward pass time 13.59 +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 298 : backward pass time 10.13 +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 298 : optimizer step time 1.56 +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 298-------------------- +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:52:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228426 41.344330 41.175629 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.226074 67.421875 67.160156 +util 90.500000 91.000000 89.000000 +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 299 : avg iteration time 46.35 seconds +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 299 : data loading time 0.00 +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 299 : avg forward pass time 12.26 +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 299 : backward pass time 10.14 +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 299 : optimizer step time 1.56 +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 299-------------------- +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:53:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.227979 41.342529 41.175629 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.804688 107.603516 +util 68.000000 69.000000 67.000000 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 300 : avg iteration time 162.10 seconds +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 300 : data loading time 0.00 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 300 : avg forward pass time 72.44 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 300 : backward pass time 8.44 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 300 : optimizer step time 0.40 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 300-------------------- +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6230 iter count: 1.0 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6230 iter count: 1.0 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6230 iter count: 1.0 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6230iter count: 1.0 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 18:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 18:56:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.67 seconds +[May 1, 2026 - 18:56:38 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 300 +[May 1, 2026 - 18:56:38 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000300 +[May 1, 2026 - 18:57:16 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 300 +[May 1, 2026 - 18:57:16 | INFO | fastgen.trainer:validate:358 ] Validation iteration 300 +[May 1, 2026 - 18:57:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:57:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.2s +[May 1, 2026 - 18:57:22 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.517 +[May 1, 2026 - 18:57:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:57:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:57:27 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.985 +[May 1, 2026 - 18:57:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:57:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:57:32 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.744 +[May 1, 2026 - 18:57:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:57:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:57:37 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.013 +[May 1, 2026 - 18:57:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:57:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:57:42 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=5.048 +[May 1, 2026 - 18:57:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:57:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:57:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.851 +[May 1, 2026 - 18:57:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:57:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:57:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.726 +[May 1, 2026 - 18:57:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:57:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:57:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.840 +[May 1, 2026 - 18:57:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:58:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:58:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.183 +[May 1, 2026 - 18:58:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 18:58:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 18:58:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.646 +[May 1, 2026 - 18:58:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 300-------------------- +[May 1, 2026 - 18:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 18:58:24 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 300 +[May 1, 2026 - 18:58:24 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.655 (n=10) +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.230048 41.349236 41.176228 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.689453 68.750000 67.335938 +util 91.000000 92.000000 90.000000 +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 301 : avg iteration time 166.29 seconds +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 301 : data loading time 0.00 +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 301 : avg forward pass time 11.89 +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 301 : backward pass time 10.13 +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 301 : optimizer step time 1.57 +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 301-------------------- +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:59:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.230048 41.349220 41.176243 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.733398 68.691406 67.414062 +util 90.500000 91.000000 89.000000 +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 302 : avg iteration time 46.59 seconds +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 302 : data loading time 0.00 +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 302 : avg forward pass time 12.34 +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 302 : backward pass time 10.17 +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 302 : optimizer step time 1.57 +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 302-------------------- +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 18:59:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.230070 41.349220 41.176228 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 68.632812 67.453125 +util 92.500000 93.000000 91.000000 +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 303 : avg iteration time 46.43 seconds +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 303 : data loading time 0.00 +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 303 : avg forward pass time 12.27 +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 303 : backward pass time 10.17 +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 303 : optimizer step time 1.57 +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 303-------------------- +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:00:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.230096 41.349220 41.176331 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.718750 68.574219 67.433594 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 304 : avg iteration time 49.31 seconds +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 304 : data loading time 0.00 +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 304 : avg forward pass time 13.73 +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 304 : backward pass time 10.14 +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 304 : optimizer step time 1.57 +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 304-------------------- +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:01:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229050 41.345058 41.176308 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.667969 107.564453 +util 64.500000 66.000000 64.000000 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 305 : avg iteration time 167.12 seconds +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 305 : data loading time 0.00 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 305 : avg forward pass time 74.96 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 305 : backward pass time 8.40 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 305 : optimizer step time 0.39 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 305-------------------- +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6504 iter count: 1.0 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6504 iter count: 1.0 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6504 iter count: 1.0 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6504iter count: 1.0 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229057 41.345078 41.176319 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.426758 67.460938 67.414062 +util 93.500000 94.000000 92.000000 +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 306 : avg iteration time 44.77 seconds +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 306 : data loading time 0.00 +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 306 : avg forward pass time 11.47 +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 306 : backward pass time 10.11 +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 306 : optimizer step time 1.58 +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 306-------------------- +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:05:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229257 41.345825 41.176369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.456055 67.474609 67.402344 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 307 : avg iteration time 46.29 seconds +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 307 : data loading time 0.00 +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 307 : avg forward pass time 12.21 +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 307 : backward pass time 10.17 +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 307 : optimizer step time 1.56 +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 307-------------------- +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:05:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229593 41.347176 41.176365 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.387695 67.480469 67.355469 +util 93.000000 94.000000 92.000000 +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 308 : avg iteration time 46.30 seconds +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 308 : data loading time 0.00 +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 308 : avg forward pass time 12.24 +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 308 : backward pass time 10.10 +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 308 : optimizer step time 1.55 +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 308-------------------- +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:06:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229666 41.347427 41.176388 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.470703 67.494141 67.402344 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 309 : avg iteration time 46.47 seconds +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 309 : data loading time 0.00 +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 309 : avg forward pass time 12.30 +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 309 : backward pass time 10.15 +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 309 : optimizer step time 1.58 +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 309-------------------- +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:07:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229175 41.345486 41.176365 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.687500 107.603516 +util 64.500000 74.000000 56.000000 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 310 : avg iteration time 159.07 seconds +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 310 : data loading time 0.00 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 310 : avg forward pass time 70.92 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 310 : backward pass time 8.42 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 310 : optimizer step time 0.40 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 310-------------------- +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5859 iter count: 1.0 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5859 iter count: 1.0 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5859 iter count: 1.0 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5859iter count: 1.0 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:10:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229172 41.345474 41.176365 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.431641 67.597656 67.375000 +util 91.000000 92.000000 90.000000 +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 311 : avg iteration time 47.63 seconds +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 311 : data loading time 0.00 +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 311 : avg forward pass time 12.85 +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 311 : backward pass time 10.16 +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 311 : optimizer step time 1.57 +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 311-------------------- +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:10:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229239 41.345726 41.176376 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480469 67.851562 67.355469 +util 94.250000 95.000000 93.000000 +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 312 : avg iteration time 46.44 seconds +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 312 : data loading time 0.00 +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 312 : avg forward pass time 12.27 +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 312 : backward pass time 10.14 +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 312 : optimizer step time 1.57 +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 312-------------------- +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:11:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229424 41.346478 41.176365 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.724609 67.806641 67.480469 +util 97.500000 98.000000 96.000000 +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 313 : avg iteration time 46.44 seconds +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 313 : data loading time 0.00 +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 313 : avg forward pass time 12.28 +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 313 : backward pass time 10.14 +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 313 : optimizer step time 1.58 +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 313-------------------- +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:12:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229561 41.346977 41.176388 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.836914 67.871094 67.824219 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 314 : avg iteration time 43.82 seconds +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 314 : data loading time 0.00 +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 314 : avg forward pass time 10.97 +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 314 : backward pass time 10.14 +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 314 : optimizer step time 1.57 +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 314-------------------- +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:13:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228955 41.344532 41.176399 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.692871 107.726562 107.681641 +util 69.750000 71.000000 69.000000 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 315 : avg iteration time 161.63 seconds +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 315 : data loading time 0.00 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 315 : avg forward pass time 72.21 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 315 : backward pass time 8.43 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 315 : optimizer step time 0.39 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 315-------------------- +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6455 iter count: 1.0 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6455 iter count: 1.0 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6455 iter count: 1.0 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6455iter count: 1.0 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:15:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228952 41.344551 41.176365 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.904297 68.019531 67.558594 +util 92.250000 93.000000 91.000000 +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 316 : avg iteration time 47.69 seconds +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 316 : data loading time 0.00 +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 316 : avg forward pass time 12.92 +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 316 : backward pass time 10.12 +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 316 : optimizer step time 1.58 +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 316-------------------- +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:16:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229072 41.345028 41.176380 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.899414 68.019531 67.539062 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 317 : avg iteration time 43.68 seconds +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 317 : data loading time 0.00 +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 317 : avg forward pass time 10.92 +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 317 : backward pass time 10.14 +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 317 : optimizer step time 1.59 +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 317-------------------- +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:17:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229260 41.345776 41.176365 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.879883 68.019531 67.460938 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 318 : avg iteration time 46.44 seconds +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 318 : data loading time 0.00 +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 318 : avg forward pass time 12.29 +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 318 : backward pass time 10.14 +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 318 : optimizer step time 1.56 +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 318-------------------- +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:18:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229298 41.345894 41.176403 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.909180 68.019531 67.578125 +util 91.750000 92.000000 91.000000 +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 319 : avg iteration time 43.68 seconds +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 319 : data loading time 0.00 +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 319 : avg forward pass time 10.91 +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 319 : backward pass time 10.14 +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 319 : optimizer step time 1.57 +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 319-------------------- +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:18:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229319 41.345905 41.176365 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.751465 107.759766 107.726562 +util 68.000000 69.000000 67.000000 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 320 : avg iteration time 164.30 seconds +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 320 : data loading time 0.00 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 320 : avg forward pass time 73.53 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 320 : backward pass time 8.45 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 320 : optimizer step time 0.40 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 320-------------------- +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6924 iter count: 1.0 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6924 iter count: 1.0 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6924 iter count: 1.0 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6924iter count: 1.0 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:21:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229053 41.344875 41.176365 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.485352 67.812500 67.375000 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 321 : avg iteration time 44.80 seconds +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 321 : data loading time 0.00 +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 321 : avg forward pass time 11.47 +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 321 : backward pass time 10.13 +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 321 : optimizer step time 1.58 +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 321-------------------- +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:22:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229060 41.344879 41.176388 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.416992 67.773438 67.296875 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 322 : avg iteration time 49.00 seconds +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 322 : data loading time 0.00 +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 322 : avg forward pass time 13.59 +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 322 : backward pass time 10.11 +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 322 : optimizer step time 1.56 +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 322-------------------- +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:23:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229124 41.345131 41.176365 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.470703 67.988281 67.296875 +util 98.000000 99.000000 97.000000 +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 323 : avg iteration time 49.18 seconds +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 323 : data loading time 0.00 +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 323 : avg forward pass time 13.65 +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 323 : backward pass time 10.14 +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 323 : optimizer step time 1.58 +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 323-------------------- +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:23:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229181 41.345360 41.176365 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412109 67.753906 67.296875 +util 97.750000 98.000000 97.000000 +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 324 : avg iteration time 43.79 seconds +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 324 : data loading time 0.00 +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 324 : avg forward pass time 10.96 +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 324 : backward pass time 10.15 +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 324 : optimizer step time 1.58 +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 324-------------------- +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:24:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229244 41.345608 41.176369 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.717285 107.740234 107.648438 +util 62.500000 64.000000 62.000000 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 325 : avg iteration time 158.97 seconds +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 325 : data loading time 0.00 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 325 : avg forward pass time 70.86 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 325 : backward pass time 8.43 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 325 : optimizer step time 0.40 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 325-------------------- +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5742 iter count: 1.0 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5742 iter count: 1.0 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5742 iter count: 1.0 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5742iter count: 1.0 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:27:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229199 41.345425 41.176369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.420898 67.500000 67.394531 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 326 : avg iteration time 47.47 seconds +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 326 : data loading time 0.00 +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 326 : avg forward pass time 12.81 +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 326 : backward pass time 10.12 +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 326 : optimizer step time 1.58 +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 326-------------------- +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:28:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229211 41.345425 41.176392 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.743164 67.824219 67.500000 +util 90.000000 99.000000 86.000000 +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 327 : avg iteration time 49.25 seconds +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 327 : data loading time 0.00 +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 327 : avg forward pass time 13.69 +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 327 : backward pass time 10.15 +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 327 : optimizer step time 1.58 +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 327-------------------- +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:28:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229210 41.345425 41.176388 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.480469 67.335938 +util 98.000000 99.000000 97.000000 +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 328 : avg iteration time 49.14 seconds +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 328 : data loading time 0.00 +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 328 : avg forward pass time 13.63 +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 328 : backward pass time 10.16 +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 328 : optimizer step time 1.57 +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 328-------------------- +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:29:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229207 41.345425 41.176369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.480469 67.355469 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 329 : avg iteration time 46.41 seconds +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 329 : data loading time 0.00 +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 329 : avg forward pass time 12.27 +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 329 : backward pass time 10.14 +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 329 : optimizer step time 1.57 +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 329-------------------- +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:30:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229212 41.345425 41.176384 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.668457 107.687500 107.662109 +util 64.500000 74.000000 56.000000 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 330 : avg iteration time 167.08 seconds +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 330 : data loading time 0.00 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 330 : avg forward pass time 74.93 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 330 : backward pass time 8.41 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 330 : optimizer step time 0.39 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 330-------------------- +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6406 iter count: 1.0 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6406 iter count: 1.0 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6406 iter count: 1.0 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6406iter count: 1.0 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:33:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229219 41.345440 41.176369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.509277 67.968750 67.355469 +util 92.500000 93.000000 92.000000 +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 331 : avg iteration time 47.46 seconds +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 331 : data loading time 0.00 +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 331 : avg forward pass time 12.81 +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 331 : backward pass time 10.13 +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 331 : optimizer step time 1.57 +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 331-------------------- +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:34:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229206 41.345417 41.176369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.519043 67.949219 67.375000 +util 95.000000 96.000000 94.000000 +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 332 : avg iteration time 49.23 seconds +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 332 : data loading time 0.00 +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 332 : avg forward pass time 13.68 +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 332 : backward pass time 10.14 +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 332 : optimizer step time 1.57 +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 332-------------------- +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:34:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229206 41.345417 41.176369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.475098 67.949219 67.316406 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 333 : avg iteration time 43.66 seconds +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 333 : data loading time 0.00 +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 333 : avg forward pass time 10.89 +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 333 : backward pass time 10.14 +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 333 : optimizer step time 1.59 +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 333-------------------- +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:35:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229270 41.345673 41.176369 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.602051 67.988281 67.472656 +util 89.750000 91.000000 89.000000 +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 334 : avg iteration time 49.09 seconds +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 334 : data loading time 0.00 +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 334 : avg forward pass time 13.62 +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 334 : backward pass time 10.12 +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 334 : optimizer step time 1.58 +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 334-------------------- +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:36:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229283 41.345688 41.176384 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.770996 107.804688 107.759766 +util 57.750000 59.000000 57.000000 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 335 : avg iteration time 164.20 seconds +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 335 : data loading time 0.00 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 335 : avg forward pass time 73.49 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 335 : backward pass time 8.41 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 335 : optimizer step time 0.40 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 335-------------------- +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6445 iter count: 1.0 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6445 iter count: 1.0 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6445 iter count: 1.0 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6445iter count: 1.0 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:39:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229149 41.345139 41.176392 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.460449 67.474609 67.421875 +util 96.500000 97.000000 96.000000 +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 336 : avg iteration time 47.48 seconds +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 336 : data loading time 0.00 +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 336 : avg forward pass time 12.81 +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 336 : backward pass time 10.15 +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 336 : optimizer step time 1.56 +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 336-------------------- +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:40:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229141 41.345146 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362793 67.500000 67.316406 +util 94.500000 95.000000 94.000000 +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 337 : avg iteration time 43.98 seconds +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 337 : data loading time 0.00 +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 337 : avg forward pass time 11.05 +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 337 : backward pass time 10.15 +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 337 : optimizer step time 1.57 +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 337-------------------- +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:40:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229164 41.345146 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362793 67.500000 67.316406 +util 92.750000 93.000000 92.000000 +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 338 : avg iteration time 49.18 seconds +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 338 : data loading time 0.00 +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 338 : avg forward pass time 13.65 +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 338 : backward pass time 10.15 +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 338 : optimizer step time 1.56 +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 338-------------------- +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:41:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229430 41.346142 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.323730 67.343750 67.316406 +util 96.500000 97.000000 96.000000 +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 339 : avg iteration time 46.44 seconds +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 339 : data loading time 0.00 +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 339 : avg forward pass time 12.28 +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 339 : backward pass time 10.15 +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 339 : optimizer step time 1.58 +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 339-------------------- +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:42:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228954 41.344296 41.176373 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.662109 107.628906 +util 60.500000 62.000000 60.000000 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 340 : avg iteration time 158.96 seconds +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 340 : data loading time 0.00 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 340 : avg forward pass time 70.87 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 340 : backward pass time 8.43 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 340 : optimizer step time 0.41 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 340-------------------- +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6396 iter count: 1.0 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6396 iter count: 1.0 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6396 iter count: 1.0 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6396iter count: 1.0 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:44:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228956 41.344280 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.572266 67.609375 67.460938 +util 93.750000 95.000000 93.000000 +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 341 : avg iteration time 44.81 seconds +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 341 : data loading time 0.00 +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 341 : avg forward pass time 11.48 +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 341 : backward pass time 10.11 +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 341 : optimizer step time 1.58 +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 341-------------------- +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:45:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228972 41.344296 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.723633 67.804688 67.480469 +util 89.500000 90.000000 89.000000 +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 342 : avg iteration time 43.75 seconds +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 342 : data loading time 0.00 +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 342 : avg forward pass time 10.94 +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 342 : backward pass time 10.18 +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 342 : optimizer step time 1.57 +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 342-------------------- +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:46:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228957 41.344280 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.738281 67.824219 67.480469 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 343 : avg iteration time 46.73 seconds +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 343 : data loading time 0.00 +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 343 : avg forward pass time 12.41 +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 343 : backward pass time 10.16 +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 343 : optimizer step time 1.59 +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 343-------------------- +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:47:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229155 41.345039 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.782227 67.882812 67.480469 +util 91.750000 93.000000 91.000000 +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 344 : avg iteration time 49.10 seconds +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 344 : data loading time 0.00 +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 344 : avg forward pass time 13.62 +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 344 : backward pass time 10.15 +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 344 : optimizer step time 1.57 +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 344-------------------- +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:48:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228895 41.343990 41.176373 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.756348 107.759766 107.746094 +util 65.250000 67.000000 64.000000 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 345 : avg iteration time 159.06 seconds +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 345 : data loading time 0.00 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 345 : avg forward pass time 70.92 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 345 : backward pass time 8.41 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 345 : optimizer step time 0.40 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 345-------------------- +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5801 iter count: 1.0 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5801 iter count: 1.0 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5801 iter count: 1.0 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5801iter count: 1.0 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:50:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228885 41.343990 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.728516 67.804688 67.500000 +util 98.000000 99.000000 97.000000 +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 346 : avg iteration time 50.42 seconds +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 346 : data loading time 0.00 +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 346 : avg forward pass time 14.26 +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 346 : backward pass time 10.17 +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 346 : optimizer step time 1.58 +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 346-------------------- +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:51:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228954 41.344242 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.699219 67.824219 67.324219 +util 90.750000 92.000000 90.000000 +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 347 : avg iteration time 46.53 seconds +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 347 : data loading time 0.00 +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 347 : avg forward pass time 12.31 +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 347 : backward pass time 10.15 +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 347 : optimizer step time 1.58 +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 347-------------------- +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:52:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228951 41.344242 41.176373 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.689453 67.804688 67.343750 +util 90.000000 91.000000 89.000000 +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 348 : avg iteration time 49.38 seconds +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 348 : data loading time 0.00 +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 348 : avg forward pass time 13.75 +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 348 : backward pass time 10.14 +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 348 : optimizer step time 1.58 +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 348-------------------- +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228964 41.344242 41.176376 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.728516 67.804688 67.500000 +util 96.750000 98.000000 96.000000 +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 349 : avg iteration time 43.68 seconds +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 349 : data loading time 0.00 +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 349 : avg forward pass time 10.91 +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 349 : backward pass time 10.15 +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 349 : optimizer step time 1.58 +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 349-------------------- +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:53:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228903 41.344021 41.176376 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.667969 107.583984 +util 69.750000 71.000000 69.000000 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 350 : avg iteration time 159.26 seconds +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 350 : data loading time 0.00 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 350 : avg forward pass time 71.04 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 350 : backward pass time 8.42 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 350 : optimizer step time 0.39 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 350-------------------- +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5820 iter count: 1.0 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5820 iter count: 1.0 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5820 iter count: 1.0 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5820iter count: 1.0 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:56:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228895 41.343998 41.176380 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407227 67.558594 67.355469 +util 89.750000 91.000000 89.000000 +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 351 : avg iteration time 47.66 seconds +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 351 : data loading time 0.00 +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 351 : avg forward pass time 12.88 +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 351 : backward pass time 10.17 +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 351 : optimizer step time 1.58 +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 351-------------------- +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:57:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228902 41.344017 41.176376 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.436523 67.558594 67.394531 +util 96.750000 98.000000 96.000000 +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 352 : avg iteration time 46.48 seconds +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 352 : data loading time 0.00 +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 352 : avg forward pass time 12.29 +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 352 : backward pass time 10.15 +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 352 : optimizer step time 1.58 +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 352-------------------- +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:58:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228891 41.343998 41.176376 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.753906 67.826172 67.539062 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 353 : avg iteration time 49.05 seconds +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 353 : data loading time 0.00 +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 353 : avg forward pass time 13.59 +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 353 : backward pass time 10.15 +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 353 : optimizer step time 1.57 +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 353-------------------- +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:58:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228891 41.343998 41.176376 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729492 67.845703 67.382812 +util 92.500000 93.000000 92.000000 +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 354 : avg iteration time 43.69 seconds +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 354 : data loading time 0.00 +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 354 : avg forward pass time 10.91 +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 354 : backward pass time 10.14 +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 354 : optimizer step time 1.57 +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 354-------------------- +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 19:59:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228718 41.343250 41.176384 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.707031 107.623047 +util 63.000000 64.000000 62.000000 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 355 : avg iteration time 162.02 seconds +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 355 : data loading time 0.00 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 355 : avg forward pass time 72.42 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 355 : backward pass time 8.41 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 355 : optimizer step time 0.39 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 355-------------------- +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5889 iter count: 1.0 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5889 iter count: 1.0 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5889 iter count: 1.0 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5889iter count: 1.0 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:02:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228711 41.343246 41.176376 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.188477 68.275391 67.929688 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 356 : avg iteration time 47.69 seconds +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 356 : data loading time 0.00 +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 356 : avg forward pass time 12.91 +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 356 : backward pass time 10.17 +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 356 : optimizer step time 1.55 +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 356-------------------- +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:03:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228905 41.344017 41.176388 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.763672 67.767578 67.753906 +util 98.000000 99.000000 97.000000 +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 357 : avg iteration time 43.75 seconds +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 357 : data loading time 0.00 +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 357 : avg forward pass time 10.94 +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 357 : backward pass time 10.14 +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 357 : optimizer step time 1.58 +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 357-------------------- +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:03:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228900 41.343994 41.176403 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.120117 68.255859 67.714844 +util 89.750000 91.000000 89.000000 +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 358 : avg iteration time 43.70 seconds +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 358 : data loading time 0.00 +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 358 : avg forward pass time 10.93 +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 358 : backward pass time 10.11 +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 358 : optimizer step time 1.57 +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 358-------------------- +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:04:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228901 41.343994 41.176399 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.861328 67.968750 67.824219 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 359 : avg iteration time 43.75 seconds +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 359 : data loading time 0.00 +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 359 : avg forward pass time 10.92 +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 359 : backward pass time 10.18 +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 359 : optimizer step time 1.59 +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 359-------------------- +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:05:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228597 41.342781 41.176395 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.663574 107.667969 107.662109 +util 64.250000 74.000000 55.000000 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 360 : avg iteration time 161.61 seconds +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 360 : data loading time 0.00 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 360 : avg forward pass time 72.21 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 360 : backward pass time 8.41 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 360 : optimizer step time 0.39 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 360-------------------- +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5781 iter count: 1.0 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5781 iter count: 1.0 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5781 iter count: 1.0 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5781iter count: 1.0 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:08:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228596 41.342781 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.504883 67.890625 67.375000 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 361 : avg iteration time 47.94 seconds +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 361 : data loading time 0.00 +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 361 : avg forward pass time 13.04 +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 361 : backward pass time 10.14 +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 361 : optimizer step time 1.58 +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 361-------------------- +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:08:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228838 41.343758 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.514648 67.871094 67.394531 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 362 : avg iteration time 43.61 seconds +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 362 : data loading time 0.00 +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 362 : avg forward pass time 10.87 +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 362 : backward pass time 10.14 +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 362 : optimizer step time 1.58 +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 362-------------------- +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:09:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228965 41.344261 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.490234 67.773438 67.394531 +util 98.500000 99.000000 98.000000 +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 363 : avg iteration time 46.58 seconds +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 363 : data loading time 0.00 +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 363 : avg forward pass time 12.33 +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 363 : backward pass time 10.19 +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 363 : optimizer step time 1.56 +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 363-------------------- +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:10:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229095 41.344784 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.563477 67.890625 67.453125 +util 93.500000 99.000000 88.000000 +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 364 : avg iteration time 44.07 seconds +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 364 : data loading time 0.00 +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 364 : avg forward pass time 11.12 +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 364 : backward pass time 10.12 +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 364 : optimizer step time 1.57 +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 364-------------------- +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:11:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228554 41.342613 41.176395 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.746094 107.623047 +util 60.750000 62.000000 60.000000 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 365 : avg iteration time 161.46 seconds +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 365 : data loading time 0.00 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 365 : avg forward pass time 72.13 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 365 : backward pass time 8.43 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 365 : optimizer step time 0.40 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 365-------------------- +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6357 iter count: 1.0 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6357 iter count: 1.0 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6357 iter count: 1.0 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6357iter count: 1.0 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:13:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228551 41.342590 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402344 67.480469 67.375000 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 366 : avg iteration time 47.69 seconds +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 366 : data loading time 0.00 +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 366 : avg forward pass time 12.91 +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 366 : backward pass time 10.14 +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 366 : optimizer step time 1.57 +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 366-------------------- +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:14:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228685 41.343113 41.176399 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.387695 67.480469 67.355469 +util 93.000000 94.000000 92.000000 +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 367 : avg iteration time 46.52 seconds +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 367 : data loading time 0.00 +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 367 : avg forward pass time 12.34 +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 367 : backward pass time 10.16 +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 367 : optimizer step time 1.56 +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 367-------------------- +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:15:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228995 41.344349 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.382812 67.460938 67.355469 +util 98.500000 99.000000 98.000000 +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 368 : avg iteration time 49.06 seconds +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 368 : data loading time 0.00 +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 368 : avg forward pass time 13.60 +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 368 : backward pass time 10.17 +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 368 : optimizer step time 1.56 +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 368-------------------- +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:16:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229127 41.344868 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407227 67.558594 67.355469 +util 90.500000 91.000000 90.000000 +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 369 : avg iteration time 49.31 seconds +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 369 : data loading time 0.00 +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 369 : avg forward pass time 13.72 +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 369 : backward pass time 10.14 +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 369 : optimizer step time 1.57 +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 369-------------------- +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:16:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228678 41.343071 41.176395 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.648438 107.564453 +util 62.250000 63.000000 61.000000 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 370 : avg iteration time 164.63 seconds +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 370 : data loading time 0.00 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 370 : avg forward pass time 73.71 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 370 : backward pass time 8.42 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 370 : optimizer step time 0.40 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 370-------------------- +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5986 iter count: 1.0 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5986 iter count: 1.0 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5986 iter count: 1.0 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5986iter count: 1.0 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:19:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228692 41.343086 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.485840 67.636719 67.435547 +util 90.500000 91.000000 90.000000 +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 371 : avg iteration time 44.96 seconds +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 371 : data loading time 0.00 +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 371 : avg forward pass time 11.55 +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 371 : backward pass time 10.14 +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 371 : optimizer step time 1.58 +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 371-------------------- +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:20:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228875 41.343834 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.427246 67.578125 67.376953 +util 90.500000 91.000000 90.000000 +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 372 : avg iteration time 43.80 seconds +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 372 : data loading time 0.00 +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 372 : avg forward pass time 10.98 +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 372 : backward pass time 10.11 +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 372 : optimizer step time 1.57 +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 372-------------------- +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:21:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229141 41.344887 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.778809 67.845703 67.578125 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 373 : avg iteration time 49.15 seconds +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 373 : data loading time 0.00 +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 373 : avg forward pass time 13.64 +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 373 : backward pass time 10.16 +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 373 : optimizer step time 1.57 +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 373-------------------- +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:22:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229137 41.344845 41.176418 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.558594 67.455078 +util 96.250000 97.000000 95.000000 +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 374 : avg iteration time 46.52 seconds +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 374 : data loading time 0.00 +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 374 : avg forward pass time 12.33 +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 374 : backward pass time 10.14 +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 374 : optimizer step time 1.58 +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 374-------------------- +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:22:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229252 41.345341 41.176395 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.687500 107.623047 +util 62.000000 64.000000 60.000000 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 375 : avg iteration time 162.17 seconds +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 375 : data loading time 0.00 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 375 : avg forward pass time 72.48 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 375 : backward pass time 8.41 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 375 : optimizer step time 0.41 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 375-------------------- +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5996 iter count: 1.0 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5996 iter count: 1.0 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5996 iter count: 1.0 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5996iter count: 1.0 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:25:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228936 41.344059 41.176395 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.804688 67.656250 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 376 : avg iteration time 47.46 seconds +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 376 : data loading time 0.00 +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 376 : avg forward pass time 12.82 +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 376 : backward pass time 10.13 +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 376 : optimizer step time 1.57 +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 376-------------------- +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:26:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.228958 41.344059 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.804688 67.656250 +util 92.000000 93.000000 91.000000 +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 377 : avg iteration time 46.52 seconds +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 377 : data loading time 0.00 +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 377 : avg forward pass time 12.30 +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 377 : backward pass time 10.18 +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 377 : optimizer step time 1.58 +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 377-------------------- +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:27:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229554 41.346432 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.578125 67.414062 +util 98.250000 99.000000 97.000000 +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 378 : avg iteration time 46.46 seconds +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 378 : data loading time 0.00 +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 378 : avg forward pass time 12.30 +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 378 : backward pass time 10.14 +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 378 : optimizer step time 1.58 +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 378-------------------- +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:27:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229554 41.346439 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.597656 67.414062 +util 92.750000 94.000000 92.000000 +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 379 : avg iteration time 46.33 seconds +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 379 : data loading time 0.00 +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 379 : avg forward pass time 12.24 +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 379 : backward pass time 10.16 +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 379 : optimizer step time 1.57 +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 379-------------------- +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:28:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229567 41.346493 41.176502 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.712402 107.720703 107.687500 +util 68.250000 70.000000 67.000000 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 380 : avg iteration time 161.68 seconds +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 380 : data loading time 0.00 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 380 : avg forward pass time 72.24 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 380 : backward pass time 8.42 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 380 : optimizer step time 0.40 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 380-------------------- +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6074 iter count: 1.0 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6074 iter count: 1.0 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6074 iter count: 1.0 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6074iter count: 1.0 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:31:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229568 41.346493 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412109 67.519531 67.375000 +util 90.000000 91.000000 89.000000 +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 381 : avg iteration time 47.69 seconds +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 381 : data loading time 0.00 +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 381 : avg forward pass time 12.90 +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 381 : backward pass time 10.15 +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 381 : optimizer step time 1.58 +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 381-------------------- +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:32:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229567 41.346493 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412109 67.460938 67.394531 +util 93.500000 100.000000 87.000000 +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 382 : avg iteration time 49.17 seconds +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 382 : data loading time 0.00 +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 382 : avg forward pass time 13.65 +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 382 : backward pass time 10.14 +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 382 : optimizer step time 1.57 +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 382-------------------- +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:32:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229577 41.346519 41.176517 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412109 67.460938 67.394531 +util 96.750000 100.000000 89.000000 +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 383 : avg iteration time 43.79 seconds +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 383 : data loading time 0.00 +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 383 : avg forward pass time 10.94 +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 383 : backward pass time 10.14 +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 383 : optimizer step time 1.58 +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 383-------------------- +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:33:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229591 41.346592 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412109 67.460938 67.394531 +util 96.250000 97.000000 95.000000 +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 384 : avg iteration time 49.72 seconds +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 384 : data loading time 0.00 +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 384 : avg forward pass time 13.93 +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 384 : backward pass time 10.13 +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 384 : optimizer step time 1.57 +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 384-------------------- +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:34:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229433 41.345936 41.176502 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.746094 107.603516 +util 66.500000 75.000000 56.000000 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 385 : avg iteration time 161.71 seconds +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 385 : data loading time 0.00 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 385 : avg forward pass time 72.21 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 385 : backward pass time 8.45 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 385 : optimizer step time 0.40 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 385-------------------- +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5869 iter count: 1.0 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5869 iter count: 1.0 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5869 iter count: 1.0 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5869iter count: 1.0 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:37:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229428 41.345936 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.406738 67.617188 67.335938 +util 95.250000 96.000000 94.000000 +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 386 : avg iteration time 50.08 seconds +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 386 : data loading time 0.00 +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 386 : avg forward pass time 14.10 +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 386 : backward pass time 10.17 +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 386 : optimizer step time 1.57 +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 386-------------------- +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:37:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229428 41.345936 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.528809 67.695312 67.472656 +util 95.500000 97.000000 94.000000 +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 387 : avg iteration time 46.41 seconds +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 387 : data loading time 0.00 +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 387 : avg forward pass time 12.28 +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 387 : backward pass time 10.15 +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 387 : optimizer step time 1.57 +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 387-------------------- +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:38:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229440 41.345955 41.176514 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.475098 67.539062 67.453125 +util 89.500000 91.000000 88.000000 +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 388 : avg iteration time 46.53 seconds +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 388 : data loading time 0.00 +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 388 : avg forward pass time 12.33 +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 388 : backward pass time 10.15 +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 388 : optimizer step time 1.57 +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 388-------------------- +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:39:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229527 41.346321 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499512 67.636719 67.453125 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 389 : avg iteration time 49.32 seconds +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 389 : data loading time 0.00 +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 389 : avg forward pass time 13.73 +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 389 : backward pass time 10.14 +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 389 : optimizer step time 1.58 +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 389-------------------- +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:40:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229298 41.345348 41.176525 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.648926 107.662109 107.609375 +util 63.500000 65.000000 62.000000 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 390 : avg iteration time 174.75 seconds +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 390 : data loading time 0.00 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 390 : avg forward pass time 78.26 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 390 : backward pass time 8.42 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 390 : optimizer step time 0.40 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 390-------------------- +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6250 iter count: 1.0 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6250 iter count: 1.0 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6250 iter count: 1.0 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6250iter count: 1.0 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:43:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229282 41.345348 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.460449 67.656250 67.394531 +util 90.750000 92.000000 89.000000 +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 391 : avg iteration time 49.50 seconds +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 391 : data loading time 0.00 +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 391 : avg forward pass time 13.32 +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 391 : backward pass time 10.15 +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 391 : optimizer step time 1.70 +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 391-------------------- +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:44:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229348 41.345589 41.176525 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455566 67.636719 67.394531 +util 88.500000 99.000000 84.000000 +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 392 : avg iteration time 51.83 seconds +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 392 : data loading time 0.00 +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 392 : avg forward pass time 14.93 +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 392 : backward pass time 10.17 +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 392 : optimizer step time 1.61 +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 392-------------------- +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:44:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.229342 41.345589 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499512 67.636719 67.453125 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 393 : avg iteration time 50.94 seconds +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 393 : data loading time 0.00 +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 393 : avg forward pass time 14.38 +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 393 : backward pass time 10.13 +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 393 : optimizer step time 1.58 +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 393-------------------- +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:45:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225670 41.330898 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484863 67.636719 67.433594 +util 97.000000 98.000000 96.000000 +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 394 : avg iteration time 44.58 seconds +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 394 : data loading time 0.00 +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 394 : avg forward pass time 10.92 +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 394 : backward pass time 10.82 +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 394 : optimizer step time 1.77 +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 394-------------------- +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:46:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225729 41.331066 41.176525 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.683105 107.746094 107.662109 +util 64.750000 66.000000 63.000000 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 395 : avg iteration time 166.95 seconds +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 395 : data loading time 0.00 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 395 : avg forward pass time 73.48 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 395 : backward pass time 9.26 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 395 : optimizer step time 0.39 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 395-------------------- +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5625 iter count: 1.0 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5625 iter count: 1.0 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5625 iter count: 1.0 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5625iter count: 1.0 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:49:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225640 41.330750 41.176525 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.993164 68.119141 67.617188 +util 93.750000 95.000000 93.000000 +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 396 : avg iteration time 48.09 seconds +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 396 : data loading time 0.00 +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 396 : avg forward pass time 13.07 +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 396 : backward pass time 10.17 +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 396 : optimizer step time 1.66 +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 396-------------------- +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:50:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225636 41.330757 41.176502 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.939453 68.060547 67.578125 +util 94.750000 99.000000 82.000000 +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 397 : avg iteration time 50.63 seconds +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 397 : data loading time 0.00 +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 397 : avg forward pass time 14.16 +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 397 : backward pass time 10.45 +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 397 : optimizer step time 1.60 +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 397-------------------- +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:50:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225639 41.330738 41.176529 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.095703 68.138672 67.968750 +util 97.500000 99.000000 96.000000 +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 398 : avg iteration time 49.32 seconds +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 398 : data loading time 0.00 +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 398 : avg forward pass time 13.71 +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 398 : backward pass time 10.16 +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 398 : optimizer step time 1.58 +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 398-------------------- +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:51:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225634 41.330738 41.176506 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.051758 68.138672 67.792969 +util 86.250000 92.000000 72.000000 +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 399 : avg iteration time 49.49 seconds +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 399 : data loading time 0.00 +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 399 : avg forward pass time 13.68 +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 399 : backward pass time 10.18 +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 399 : optimizer step time 1.75 +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 399-------------------- +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225641 41.330753 41.176506 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.687500 107.623047 +util 62.500000 64.000000 61.000000 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 400 : avg iteration time 163.30 seconds +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 400 : data loading time 0.00 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 400 : avg forward pass time 73.04 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 400 : backward pass time 8.41 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 400 : optimizer step time 0.43 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 400-------------------- +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5869 iter count: 1.0 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5869 iter count: 1.0 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5869 iter count: 1.0 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5869iter count: 1.0 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 20:55:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.65 seconds +[May 1, 2026 - 20:55:35 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 400 +[May 1, 2026 - 20:55:35 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000400 +[May 1, 2026 - 20:56:16 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 400 +[May 1, 2026 - 20:56:17 | INFO | fastgen.trainer:validate:358 ] Validation iteration 400 +[May 1, 2026 - 20:56:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:56:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.2s +[May 1, 2026 - 20:56:22 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.576 +[May 1, 2026 - 20:56:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:56:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:56:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.697 +[May 1, 2026 - 20:56:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:56:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:56:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.929 +[May 1, 2026 - 20:56:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:56:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:56:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.358 +[May 1, 2026 - 20:56:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:56:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:56:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.803 +[May 1, 2026 - 20:56:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:56:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:56:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.880 +[May 1, 2026 - 20:56:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:56:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:56:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.213 +[May 1, 2026 - 20:56:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:56:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:56:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.129 +[May 1, 2026 - 20:56:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:57:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:57:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.191 +[May 1, 2026 - 20:57:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 20:57:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 20:57:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.260 +[May 1, 2026 - 20:57:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 400-------------------- +[May 1, 2026 - 20:57:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 20:57:25 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 400 +[May 1, 2026 - 20:57:25 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.504 (n=10) +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.226905 41.335350 41.176662 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.109375 69.082031 67.785156 +util 95.750000 97.000000 94.000000 +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 401 : avg iteration time 170.40 seconds +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 401 : data loading time 0.00 +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 401 : avg forward pass time 11.97 +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 401 : backward pass time 10.15 +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 401 : optimizer step time 1.57 +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 401-------------------- +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:58:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.226905 41.335354 41.176685 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.811523 68.945312 67.433594 +util 89.750000 91.000000 88.000000 +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 402 : avg iteration time 43.92 seconds +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 402 : data loading time 0.00 +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 402 : avg forward pass time 11.00 +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 402 : backward pass time 10.19 +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 402 : optimizer step time 1.57 +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 402-------------------- +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:58:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.226897 41.335350 41.176662 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.840820 69.062500 67.433594 +util 88.000000 89.000000 86.000000 +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 403 : avg iteration time 46.59 seconds +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 403 : data loading time 0.00 +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 403 : avg forward pass time 12.32 +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 403 : backward pass time 10.15 +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 403 : optimizer step time 1.59 +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 403-------------------- +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 20:59:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.226480 41.333611 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.811523 69.062500 67.394531 +util 92.250000 93.000000 91.000000 +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 404 : avg iteration time 46.64 seconds +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 404 : data loading time 0.00 +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 404 : avg forward pass time 12.40 +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 404 : backward pass time 10.12 +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 404 : optimizer step time 1.57 +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 404-------------------- +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:00:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225862 41.331120 41.176750 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.609375 107.583984 +util 69.750000 71.000000 68.000000 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 405 : avg iteration time 164.91 seconds +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 405 : data loading time 0.00 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 405 : avg forward pass time 73.77 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 405 : backward pass time 8.48 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 405 : optimizer step time 0.40 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 405-------------------- +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5918 iter count: 1.0 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5918 iter count: 1.0 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5918 iter count: 1.0 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5918iter count: 1.0 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:03:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225859 41.331104 41.176754 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.421387 67.675781 67.335938 +util 91.250000 99.000000 88.000000 +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 406 : avg iteration time 47.64 seconds +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 406 : data loading time 0.00 +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 406 : avg forward pass time 12.84 +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 406 : backward pass time 10.14 +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 406 : optimizer step time 1.56 +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 406-------------------- +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:04:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225861 41.331104 41.176754 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.792480 67.826172 67.695312 +util 93.750000 95.000000 92.000000 +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 407 : avg iteration time 46.46 seconds +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 407 : data loading time 0.00 +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 407 : avg forward pass time 12.31 +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 407 : backward pass time 10.12 +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 407 : optimizer step time 1.57 +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 407-------------------- +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:04:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225913 41.331322 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.816895 67.865234 67.675781 +util 88.750000 90.000000 87.000000 +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 408 : avg iteration time 43.94 seconds +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 408 : data loading time 0.00 +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 408 : avg forward pass time 11.04 +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 408 : backward pass time 10.13 +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 408 : optimizer step time 1.56 +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 408-------------------- +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:05:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225910 41.331322 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.812012 67.845703 67.714844 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 409 : avg iteration time 46.55 seconds +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 409 : data loading time 0.00 +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 409 : avg forward pass time 12.34 +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 409 : backward pass time 10.14 +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 409 : optimizer step time 1.58 +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 409-------------------- +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:06:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225787 41.330830 41.176731 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.673340 107.765625 107.642578 +util 63.250000 64.000000 61.000000 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 410 : avg iteration time 162.12 seconds +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 410 : data loading time 0.00 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 410 : avg forward pass time 72.43 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 410 : backward pass time 8.43 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 410 : optimizer step time 0.39 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 410-------------------- +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5508 iter count: 1.0 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5508 iter count: 1.0 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5508 iter count: 1.0 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5508iter count: 1.0 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:09:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225787 41.330830 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 67.792969 67.394531 +util 93.500000 95.000000 92.000000 +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 411 : avg iteration time 50.38 seconds +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 411 : data loading time 0.00 +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 411 : avg forward pass time 14.28 +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 411 : backward pass time 10.14 +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 411 : optimizer step time 1.56 +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 411-------------------- +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:09:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225787 41.330830 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.773438 67.355469 +util 92.000000 93.000000 91.000000 +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 412 : avg iteration time 46.59 seconds +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 412 : data loading time 0.00 +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 412 : avg forward pass time 12.37 +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 412 : backward pass time 10.15 +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 412 : optimizer step time 1.57 +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 412-------------------- +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:10:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225788 41.330830 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.812500 67.335938 +util 96.500000 98.000000 95.000000 +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 413 : avg iteration time 46.65 seconds +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 413 : data loading time 0.00 +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 413 : avg forward pass time 12.41 +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 413 : backward pass time 10.13 +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 413 : optimizer step time 1.57 +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 413-------------------- +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:11:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225789 41.330830 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.812500 67.375000 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 414 : avg iteration time 46.44 seconds +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 414 : data loading time 0.00 +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 414 : avg forward pass time 12.28 +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 414 : backward pass time 10.13 +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 414 : optimizer step time 1.58 +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 414-------------------- +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:12:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225705 41.330498 41.176731 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.765625 107.525391 +util 67.750000 70.000000 65.000000 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 415 : avg iteration time 160.01 seconds +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 415 : data loading time 0.00 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 415 : avg forward pass time 71.31 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 415 : backward pass time 8.54 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 415 : optimizer step time 0.40 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 415-------------------- +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5957 iter count: 1.0 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5957 iter count: 1.0 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5957 iter count: 1.0 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5957iter count: 1.0 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:14:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225707 41.330494 41.176743 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.431152 67.714844 67.335938 +util 97.750000 100.000000 95.000000 +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 416 : avg iteration time 47.57 seconds +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 416 : data loading time 0.00 +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 416 : avg forward pass time 12.86 +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 416 : backward pass time 10.13 +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 416 : optimizer step time 1.58 +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 416-------------------- +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:15:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225704 41.330494 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.426270 67.695312 67.335938 +util 98.500000 100.000000 97.000000 +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 417 : avg iteration time 49.46 seconds +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 417 : data loading time 0.00 +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 417 : avg forward pass time 12.19 +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 417 : backward pass time 11.76 +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 417 : optimizer step time 1.59 +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 417-------------------- +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:16:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225797 41.330864 41.176731 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411621 67.753906 67.296875 +util 99.750000 100.000000 99.000000 +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 418 : avg iteration time 55.79 seconds +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 418 : data loading time 0.00 +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 418 : avg forward pass time 15.58 +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 418 : backward pass time 11.68 +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 418 : optimizer step time 1.78 +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 418-------------------- +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:17:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225834 41.330864 41.176861 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.445801 67.714844 67.355469 +util 95.000000 99.000000 92.000000 +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 419 : avg iteration time 54.91 seconds +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 419 : data loading time 0.00 +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 419 : avg forward pass time 15.07 +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 419 : backward pass time 11.17 +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 419 : optimizer step time 1.80 +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 419-------------------- +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:18:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225726 41.330452 41.176861 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.707031 107.505859 +util 76.500000 96.000000 69.000000 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 420 : avg iteration time 185.07 seconds +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 420 : data loading time 0.00 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 420 : avg forward pass time 82.21 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 420 : backward pass time 10.26 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 420 : optimizer step time 0.39 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 420-------------------- +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6025 iter count: 1.0 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6025 iter count: 1.0 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6025 iter count: 1.0 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6025iter count: 1.0 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:21:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225742 41.330452 41.176903 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.289551 67.617188 67.179688 +util 91.250000 99.000000 87.000000 +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 421 : avg iteration time 56.28 seconds +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 421 : data loading time 0.00 +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 421 : avg forward pass time 15.81 +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 421 : backward pass time 11.78 +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 421 : optimizer step time 1.79 +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 421-------------------- +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:22:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225766 41.330463 41.176903 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.450684 67.675781 67.375000 +util 89.500000 91.000000 88.000000 +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 422 : avg iteration time 49.24 seconds +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 422 : data loading time 0.00 +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 422 : avg forward pass time 12.00 +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 422 : backward pass time 11.83 +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 422 : optimizer step time 1.60 +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 422-------------------- +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:23:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225818 41.330669 41.176903 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455566 67.695312 67.375000 +util 98.750000 99.000000 98.000000 +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 423 : avg iteration time 49.70 seconds +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 423 : data loading time 0.00 +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 423 : avg forward pass time 13.05 +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 423 : backward pass time 10.17 +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 423 : optimizer step time 1.59 +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 423-------------------- +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:23:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225994 41.331360 41.176918 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.436035 67.617188 67.375000 +util 95.250000 100.000000 92.000000 +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 424 : avg iteration time 52.12 seconds +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 424 : data loading time 0.00 +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 424 : avg forward pass time 13.68 +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 424 : backward pass time 11.80 +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 424 : optimizer step time 1.78 +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 424-------------------- +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:24:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225772 41.330460 41.176907 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.765625 107.544922 +util 71.250000 94.000000 62.000000 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 425 : avg iteration time 174.00 seconds +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 425 : data loading time 0.00 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 425 : avg forward pass time 77.15 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 425 : backward pass time 9.41 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 425 : optimizer step time 0.39 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 425-------------------- +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5449 iter count: 1.0 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5449 iter count: 1.0 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5449 iter count: 1.0 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5449iter count: 1.0 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:27:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225795 41.330460 41.176998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.406738 67.617188 67.335938 +util 97.250000 99.000000 95.000000 +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 426 : avg iteration time 57.03 seconds +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 426 : data loading time 0.00 +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 426 : avg forward pass time 16.10 +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 426 : backward pass time 11.19 +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 426 : optimizer step time 1.81 +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 426-------------------- +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:28:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225854 41.330719 41.176998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401855 67.656250 67.316406 +util 91.000000 99.000000 88.000000 +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 427 : avg iteration time 55.09 seconds +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 427 : data loading time 0.00 +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 427 : avg forward pass time 15.27 +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 427 : backward pass time 11.54 +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 427 : optimizer step time 1.57 +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 427-------------------- +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:29:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225853 41.330715 41.176998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411621 67.578125 67.355469 +util 89.250000 91.000000 87.000000 +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 428 : avg iteration time 52.02 seconds +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 428 : data loading time 0.00 +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 428 : avg forward pass time 13.55 +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 428 : backward pass time 11.29 +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 428 : optimizer step time 1.79 +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 428-------------------- +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:30:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225854 41.330719 41.176998 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396973 67.578125 67.335938 +util 97.500000 100.000000 95.000000 +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 429 : avg iteration time 52.69 seconds +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 429 : data loading time 0.00 +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 429 : avg forward pass time 13.63 +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 429 : backward pass time 11.82 +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 429 : optimizer step time 1.79 +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 429-------------------- +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:31:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225539 41.329433 41.177025 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.863281 107.583984 +util 60.250000 62.000000 59.000000 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 430 : avg iteration time 182.19 seconds +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 430 : data loading time 0.00 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 430 : avg forward pass time 81.76 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 430 : backward pass time 9.76 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 430 : optimizer step time 0.42 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 430-------------------- +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5830 iter count: 1.0 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5830 iter count: 1.0 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5830 iter count: 1.0 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5830iter count: 1.0 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:34:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225536 41.329437 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.514160 67.929688 67.375000 +util 99.750000 100.000000 99.000000 +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 431 : avg iteration time 54.49 seconds +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 431 : data loading time 0.00 +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 431 : avg forward pass time 14.53 +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 431 : backward pass time 11.81 +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 431 : optimizer step time 1.79 +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 431-------------------- +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:35:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225645 41.329872 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411621 67.519531 67.375000 +util 88.000000 90.000000 86.000000 +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 432 : avg iteration time 50.83 seconds +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 432 : data loading time 0.00 +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 432 : avg forward pass time 13.21 +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 432 : backward pass time 11.65 +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 432 : optimizer step time 1.56 +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 432-------------------- +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:36:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225967 41.331116 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.421387 67.500000 67.394531 +util 96.250000 100.000000 87.000000 +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 433 : avg iteration time 50.58 seconds +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 433 : data loading time 0.00 +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 433 : avg forward pass time 12.57 +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 433 : backward pass time 11.83 +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 433 : optimizer step time 1.79 +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 433-------------------- +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:36:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.226010 41.331333 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.436035 67.558594 67.394531 +util 94.000000 100.000000 90.000000 +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 434 : avg iteration time 52.05 seconds +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 434 : data loading time 0.00 +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 434 : avg forward pass time 13.31 +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 434 : backward pass time 11.81 +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 434 : optimizer step time 1.78 +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 434-------------------- +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:37:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225533 41.329422 41.177006 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.702637 107.882812 107.642578 +util 63.500000 67.000000 60.000000 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 435 : avg iteration time 179.60 seconds +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 435 : data loading time 0.00 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 435 : avg forward pass time 80.12 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 435 : backward pass time 9.58 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 435 : optimizer step time 0.39 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 435-------------------- +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5762 iter count: 1.0 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5762 iter count: 1.0 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5762 iter count: 1.0 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5762iter count: 1.0 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225545 41.329430 41.177025 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396484 67.753906 67.277344 +util 91.750000 93.000000 90.000000 +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 436 : avg iteration time 52.61 seconds +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 436 : data loading time 0.00 +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 436 : avg forward pass time 14.72 +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 436 : backward pass time 11.39 +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 436 : optimizer step time 1.56 +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 436-------------------- +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:41:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225607 41.329678 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.871094 67.277344 +util 89.500000 91.000000 87.000000 +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 437 : avg iteration time 55.97 seconds +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 437 : data loading time 0.00 +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 437 : avg forward pass time 15.55 +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 437 : backward pass time 11.40 +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 437 : optimizer step time 1.56 +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 437-------------------- +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:42:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225611 41.329689 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.871094 67.355469 +util 98.500000 100.000000 96.000000 +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 438 : avg iteration time 48.70 seconds +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 438 : data loading time 0.00 +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 438 : avg forward pass time 11.97 +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 438 : backward pass time 11.82 +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 438 : optimizer step time 1.79 +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 438-------------------- +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:43:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225731 41.330196 41.177025 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.469727 67.871094 67.335938 +util 92.250000 94.000000 90.000000 +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 439 : avg iteration time 50.73 seconds +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 439 : data loading time 0.00 +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 439 : avg forward pass time 13.09 +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 439 : backward pass time 11.52 +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 439 : optimizer step time 1.80 +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 439-------------------- +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:44:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225805 41.330490 41.177006 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.658691 107.707031 107.642578 +util 74.750000 95.000000 66.000000 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 440 : avg iteration time 186.47 seconds +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 440 : data loading time 0.00 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 440 : avg forward pass time 83.35 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 440 : backward pass time 9.51 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 440 : optimizer step time 0.39 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 440-------------------- +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5908 iter count: 1.0 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5908 iter count: 1.0 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5908 iter count: 1.0 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5908iter count: 1.0 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:47:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225805 41.330490 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.656250 67.394531 +util 93.750000 99.000000 90.000000 +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 441 : avg iteration time 51.14 seconds +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 441 : data loading time 0.00 +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 441 : avg forward pass time 12.91 +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 441 : backward pass time 11.81 +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 441 : optimizer step time 1.79 +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 441-------------------- +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:48:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225816 41.330513 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.636719 67.394531 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 442 : avg iteration time 51.70 seconds +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 442 : data loading time 0.00 +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 442 : avg forward pass time 13.69 +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 442 : backward pass time 11.27 +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 442 : optimizer step time 1.56 +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 442-------------------- +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:49:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225805 41.330490 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.464844 67.675781 67.394531 +util 94.250000 99.000000 91.000000 +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 443 : avg iteration time 48.50 seconds +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 443 : data loading time 0.00 +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 443 : avg forward pass time 11.84 +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 443 : backward pass time 11.83 +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 443 : optimizer step time 1.79 +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 443-------------------- +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:49:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225985 41.331223 41.177006 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.695312 67.414062 +util 92.750000 99.000000 89.000000 +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 444 : avg iteration time 51.38 seconds +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 444 : data loading time 0.00 +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 444 : avg forward pass time 13.52 +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 444 : backward pass time 11.30 +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 444 : optimizer step time 1.66 +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 444-------------------- +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:50:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225822 41.330574 41.177006 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.765625 107.583984 +util 76.750000 95.000000 69.000000 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 445 : avg iteration time 183.71 seconds +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 445 : data loading time 0.00 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 445 : avg forward pass time 82.18 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 445 : backward pass time 9.75 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 445 : optimizer step time 0.39 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 445-------------------- +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6025 iter count: 1.0 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6025 iter count: 1.0 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6025 iter count: 1.0 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6025iter count: 1.0 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:53:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225902 41.330795 41.177017 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.392578 67.792969 67.257812 +util 93.000000 99.000000 87.000000 +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 446 : avg iteration time 56.03 seconds +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 446 : data loading time 0.00 +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 446 : avg forward pass time 15.78 +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 446 : backward pass time 11.05 +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 446 : optimizer step time 1.56 +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 446-------------------- +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:54:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225896 41.330795 41.177017 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.436523 67.792969 67.316406 +util 97.750000 100.000000 95.000000 +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 447 : avg iteration time 53.32 seconds +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 447 : data loading time 0.00 +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 447 : avg forward pass time 14.25 +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 447 : backward pass time 11.82 +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 447 : optimizer step time 1.78 +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 447-------------------- +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:55:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225921 41.330799 41.177017 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.421875 67.792969 67.296875 +util 90.250000 92.000000 88.000000 +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 448 : avg iteration time 55.72 seconds +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 448 : data loading time 0.00 +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 448 : avg forward pass time 15.50 +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 448 : backward pass time 11.20 +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 448 : optimizer step time 1.72 +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 448-------------------- +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:56:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225983 41.331085 41.177017 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407227 67.792969 67.277344 +util 92.500000 94.000000 90.000000 +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 449 : avg iteration time 49.23 seconds +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 449 : data loading time 0.00 +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 449 : avg forward pass time 13.45 +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 449 : backward pass time 10.17 +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 449 : optimizer step time 1.59 +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 449-------------------- +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 21:57:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.225984 41.331074 41.177040 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.687500 107.583984 +util 57.250000 59.000000 55.000000 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 450 : avg iteration time 162.55 seconds +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 450 : data loading time 0.00 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 450 : avg forward pass time 72.60 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 450 : backward pass time 8.46 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 450 : optimizer step time 0.40 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 450-------------------- +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6035 iter count: 1.0 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6035 iter count: 1.0 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6035 iter count: 1.0 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6035iter count: 1.0 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:00:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.226040 41.331322 41.177017 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480469 67.851562 67.355469 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 451 : avg iteration time 50.37 seconds +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 451 : data loading time 0.00 +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 451 : avg forward pass time 14.24 +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 451 : backward pass time 10.14 +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 451 : optimizer step time 1.58 +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 451-------------------- +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:00:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.226046 41.331345 41.177017 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.495117 67.851562 67.375000 +util 90.750000 99.000000 87.000000 +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 452 : avg iteration time 46.41 seconds +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 452 : data loading time 0.00 +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 452 : avg forward pass time 12.29 +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 452 : backward pass time 10.13 +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 452 : optimizer step time 1.57 +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 452-------------------- +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:01:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.226046 41.331345 41.177017 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.529297 67.871094 67.414062 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 453 : avg iteration time 49.13 seconds +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 453 : data loading time 0.00 +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 453 : avg forward pass time 13.64 +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 453 : backward pass time 10.13 +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 453 : optimizer step time 1.56 +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 453-------------------- +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:02:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417165 41.527607 41.366409 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.534180 68.007812 67.375000 +util 92.500000 94.000000 90.000000 +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 454 : avg iteration time 49.13 seconds +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 454 : data loading time 0.00 +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 454 : avg forward pass time 13.06 +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 454 : backward pass time 10.14 +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 454 : optimizer step time 1.58 +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 454-------------------- +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:03:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417354 41.527657 41.366837 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.678223 107.726562 107.662109 +util 68.500000 71.000000 66.000000 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 455 : avg iteration time 159.90 seconds +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 455 : data loading time 0.00 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 455 : avg forward pass time 71.27 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 455 : backward pass time 8.42 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 455 : optimizer step time 0.40 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 455-------------------- +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5601 iter count: 1.0 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5601 iter count: 1.0 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5601 iter count: 1.0 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5601iter count: 1.0 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:06:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417953 41.528542 41.367245 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.274414 67.675781 67.140625 +util 91.750000 93.000000 90.000000 +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 456 : avg iteration time 47.89 seconds +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 456 : data loading time 0.00 +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 456 : avg forward pass time 13.00 +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 456 : backward pass time 10.14 +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 456 : optimizer step time 1.58 +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 456-------------------- +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:06:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418230 41.528828 41.367645 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.636719 67.335938 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 457 : avg iteration time 49.31 seconds +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 457 : data loading time 0.00 +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 457 : avg forward pass time 13.74 +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 457 : backward pass time 10.15 +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 457 : optimizer step time 1.56 +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 457-------------------- +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418859 41.529720 41.368038 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.656250 67.316406 +util 96.750000 100.000000 88.000000 +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 458 : avg iteration time 49.42 seconds +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 458 : data loading time 0.00 +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 458 : avg forward pass time 13.78 +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 458 : backward pass time 10.16 +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 458 : optimizer step time 1.57 +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 458-------------------- +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:08:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419106 41.530231 41.368149 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.430664 67.656250 67.355469 +util 93.500000 95.000000 91.000000 +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 459 : avg iteration time 49.28 seconds +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 459 : data loading time 0.00 +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 459 : avg forward pass time 13.67 +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 459 : backward pass time 10.15 +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 459 : optimizer step time 1.56 +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 459-------------------- +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:09:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419484 41.530449 41.369324 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.687500 107.603516 +util 63.000000 65.000000 60.000000 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 460 : avg iteration time 162.22 seconds +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 460 : data loading time 0.00 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 460 : avg forward pass time 72.48 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 460 : backward pass time 8.44 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 460 : optimizer step time 0.40 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 460-------------------- +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5498 iter count: 1.0 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5498 iter count: 1.0 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5498 iter count: 1.0 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5498iter count: 1.0 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:12:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419820 41.530155 41.369781 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407227 67.558594 67.355469 +util 91.000000 99.000000 88.000000 +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 461 : avg iteration time 45.12 seconds +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 461 : data loading time 0.00 +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 461 : avg forward pass time 11.63 +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 461 : backward pass time 10.15 +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 461 : optimizer step time 1.57 +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 461-------------------- +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.420238 41.530193 41.370296 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.519531 67.949219 67.375000 +util 92.000000 100.000000 89.000000 +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 462 : avg iteration time 43.81 seconds +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 462 : data loading time 0.00 +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 462 : avg forward pass time 10.97 +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 462 : backward pass time 10.14 +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 462 : optimizer step time 1.57 +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 462-------------------- +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:13:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.420988 41.530323 41.371220 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.529297 67.988281 67.375000 +util 96.500000 98.000000 94.000000 +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 463 : avg iteration time 46.70 seconds +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 463 : data loading time 0.00 +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 463 : avg forward pass time 12.38 +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 463 : backward pass time 10.19 +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 463 : optimizer step time 1.57 +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 463-------------------- +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:14:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.421299 41.530453 41.372200 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.524414 67.968750 67.375000 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 464 : avg iteration time 46.67 seconds +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 464 : data loading time 0.00 +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 464 : avg forward pass time 12.40 +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 464 : backward pass time 10.14 +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 464 : optimizer step time 1.58 +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 464-------------------- +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:15:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.421483 41.530907 41.372459 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.785156 107.583984 +util 67.250000 69.000000 65.000000 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 465 : avg iteration time 161.96 seconds +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 465 : data loading time 0.00 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 465 : avg forward pass time 72.29 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 465 : backward pass time 8.49 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 465 : optimizer step time 0.39 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 465-------------------- +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5200 iter count: 1.0 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5200 iter count: 1.0 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5200 iter count: 1.0 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5200iter count: 1.0 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.421565 41.530621 41.372662 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455566 67.695312 67.375000 +util 92.000000 99.000000 89.000000 +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 466 : avg iteration time 44.89 seconds +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 466 : data loading time 0.00 +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 466 : avg forward pass time 11.54 +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 466 : backward pass time 10.11 +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 466 : optimizer step time 1.57 +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 466-------------------- +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:18:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.421732 41.530743 41.372772 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.416504 67.656250 67.335938 +util 96.500000 98.000000 94.000000 +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 467 : avg iteration time 49.10 seconds +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 467 : data loading time 0.00 +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 467 : avg forward pass time 13.61 +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 467 : backward pass time 10.16 +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 467 : optimizer step time 1.58 +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 467-------------------- +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:19:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.421948 41.531010 41.373047 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.431152 67.656250 67.355469 +util 95.750000 99.000000 86.000000 +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 468 : avg iteration time 49.26 seconds +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 468 : data loading time 0.00 +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 468 : avg forward pass time 13.70 +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 468 : backward pass time 10.14 +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 468 : optimizer step time 1.59 +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 468-------------------- +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:20:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.422139 41.531612 41.373062 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440918 67.695312 67.355469 +util 96.500000 100.000000 88.000000 +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 469 : avg iteration time 46.41 seconds +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 469 : data loading time 0.00 +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 469 : avg forward pass time 12.27 +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 469 : backward pass time 10.15 +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 469 : optimizer step time 1.58 +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 469-------------------- +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:20:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.422105 41.530441 41.374069 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.662109 107.628906 +util 58.000000 61.000000 54.000000 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 470 : avg iteration time 157.24 seconds +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 470 : data loading time 0.00 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 470 : avg forward pass time 69.96 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 470 : backward pass time 8.49 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 470 : optimizer step time 0.39 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 470-------------------- +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5645 iter count: 1.0 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5645 iter count: 1.0 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5645 iter count: 1.0 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5645iter count: 1.0 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:23:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.422511 41.530457 41.374260 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.538574 68.085938 67.355469 +util 94.500000 96.000000 92.000000 +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 471 : avg iteration time 45.27 seconds +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 471 : data loading time 0.00 +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 471 : avg forward pass time 11.68 +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 471 : backward pass time 10.17 +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 471 : optimizer step time 1.58 +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 471-------------------- +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:24:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.422609 41.530476 41.374462 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.523926 68.027344 67.355469 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 472 : avg iteration time 49.46 seconds +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 472 : data loading time 0.00 +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 472 : avg forward pass time 13.81 +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 472 : backward pass time 10.13 +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 472 : optimizer step time 1.58 +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 472-------------------- +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:25:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.422932 41.530582 41.374657 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.553223 67.968750 67.414062 +util 89.250000 90.000000 88.000000 +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 473 : avg iteration time 46.68 seconds +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 473 : data loading time 0.00 +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 473 : avg forward pass time 12.39 +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 473 : backward pass time 10.16 +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 473 : optimizer step time 1.57 +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 473-------------------- +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:25:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423331 41.531643 41.374844 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.538574 67.968750 67.394531 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 474 : avg iteration time 49.11 seconds +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 474 : data loading time 0.00 +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 474 : avg forward pass time 13.63 +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 474 : backward pass time 10.14 +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 474 : optimizer step time 1.58 +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 474-------------------- +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:26:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423257 41.531364 41.374844 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.726562 107.564453 +util 68.250000 70.000000 66.000000 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 475 : avg iteration time 157.02 seconds +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 475 : data loading time 0.00 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 475 : avg forward pass time 69.90 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 475 : backward pass time 8.46 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 475 : optimizer step time 0.39 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 475-------------------- +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5547 iter count: 1.0 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5547 iter count: 1.0 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5547 iter count: 1.0 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5547iter count: 1.0 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:29:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423401 41.531506 41.374916 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.382324 67.519531 67.335938 +util 93.500000 100.000000 86.000000 +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 476 : avg iteration time 47.64 seconds +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 476 : data loading time 0.00 +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 476 : avg forward pass time 12.88 +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 476 : backward pass time 10.15 +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 476 : optimizer step time 1.59 +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 476-------------------- +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:30:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423460 41.531647 41.375038 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.392090 67.675781 67.296875 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 477 : avg iteration time 46.58 seconds +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 477 : data loading time 0.00 +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 477 : avg forward pass time 12.35 +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 477 : backward pass time 10.15 +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 477 : optimizer step time 1.56 +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 477-------------------- +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:30:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423482 41.531639 41.375038 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396973 67.519531 67.355469 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 478 : avg iteration time 46.40 seconds +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 478 : data loading time 0.00 +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 478 : avg forward pass time 12.27 +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 478 : backward pass time 10.13 +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 478 : optimizer step time 1.58 +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 478-------------------- +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:31:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423586 41.531654 41.375099 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.343262 67.539062 67.277344 +util 99.000000 100.000000 98.000000 +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 479 : avg iteration time 49.39 seconds +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 479 : data loading time 0.00 +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 479 : avg forward pass time 13.77 +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 479 : backward pass time 10.14 +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 479 : optimizer step time 1.57 +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 479-------------------- +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:32:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423684 41.531933 41.375160 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.785156 107.583984 +util 69.500000 72.000000 67.000000 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 480 : avg iteration time 162.21 seconds +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 480 : data loading time 0.00 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 480 : avg forward pass time 72.46 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 480 : backward pass time 8.45 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 480 : optimizer step time 0.39 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 480-------------------- +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5190 iter count: 1.0 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5190 iter count: 1.0 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5190 iter count: 1.0 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5190iter count: 1.0 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423675 41.531887 41.375164 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.890625 67.316406 +util 92.500000 94.000000 90.000000 +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 481 : avg iteration time 47.70 seconds +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 481 : data loading time 0.00 +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 481 : avg forward pass time 12.93 +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 481 : backward pass time 10.15 +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 481 : optimizer step time 1.57 +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 481-------------------- +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:36:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423775 41.532146 41.375198 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.503906 67.890625 67.375000 +util 91.250000 100.000000 88.000000 +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 482 : avg iteration time 49.15 seconds +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 482 : data loading time 0.00 +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 482 : avg forward pass time 13.63 +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 482 : backward pass time 10.14 +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 482 : optimizer step time 1.57 +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 482-------------------- +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:36:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423830 41.532249 41.375168 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.503906 67.890625 67.375000 +util 94.000000 96.000000 91.000000 +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 483 : avg iteration time 49.27 seconds +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 483 : data loading time 0.00 +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 483 : avg forward pass time 13.70 +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 483 : backward pass time 10.14 +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 483 : optimizer step time 1.58 +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 483-------------------- +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:37:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423881 41.532379 41.375191 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.871094 67.355469 +util 95.250000 97.000000 93.000000 +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 484 : avg iteration time 44.02 seconds +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 484 : data loading time 0.00 +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 484 : avg forward pass time 11.06 +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 484 : backward pass time 10.16 +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 484 : optimizer step time 1.57 +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 484-------------------- +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:38:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423850 41.531898 41.375439 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.804688 107.583984 +util 63.500000 65.000000 60.000000 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 485 : avg iteration time 162.39 seconds +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 485 : data loading time 0.00 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 485 : avg forward pass time 72.58 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 485 : backward pass time 8.42 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 485 : optimizer step time 0.39 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 485-------------------- +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5996 iter count: 1.0 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5996 iter count: 1.0 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5996 iter count: 1.0 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5996iter count: 1.0 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:41:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423891 41.531902 41.375534 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.426270 67.695312 67.335938 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 486 : avg iteration time 45.26 seconds +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 486 : data loading time 0.00 +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 486 : avg forward pass time 11.71 +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 486 : backward pass time 10.13 +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 486 : optimizer step time 1.58 +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 486-------------------- +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:41:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.423980 41.531937 41.375595 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372559 67.480469 67.335938 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 487 : avg iteration time 49.40 seconds +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 487 : data loading time 0.00 +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 487 : avg forward pass time 13.78 +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 487 : backward pass time 10.13 +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 487 : optimizer step time 1.56 +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 487-------------------- +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:42:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424022 41.532043 41.375641 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.426270 67.695312 67.335938 +util 91.500000 100.000000 88.000000 +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 488 : avg iteration time 46.66 seconds +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 488 : data loading time 0.00 +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 488 : avg forward pass time 12.40 +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 488 : backward pass time 10.14 +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 488 : optimizer step time 1.57 +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 488-------------------- +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:43:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424076 41.532295 41.375641 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.406738 67.675781 67.316406 +util 91.250000 93.000000 88.000000 +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 489 : avg iteration time 46.44 seconds +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 489 : data loading time 0.00 +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 489 : avg forward pass time 12.31 +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 489 : backward pass time 10.14 +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 489 : optimizer step time 1.57 +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 489-------------------- +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:44:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424021 41.531925 41.375648 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.707031 107.564453 +util 62.000000 70.000000 56.000000 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 490 : avg iteration time 165.35 seconds +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 490 : data loading time 0.00 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 490 : avg forward pass time 74.03 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 490 : backward pass time 8.43 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 490 : optimizer step time 0.39 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 490-------------------- +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5830 iter count: 1.0 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5830 iter count: 1.0 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5830 iter count: 1.0 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5830iter count: 1.0 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:46:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424129 41.531979 41.375912 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.929199 68.183594 67.843750 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 491 : avg iteration time 47.75 seconds +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 491 : data loading time 0.00 +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 491 : avg forward pass time 12.95 +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 491 : backward pass time 10.14 +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 491 : optimizer step time 1.57 +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 491-------------------- +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424290 41.532269 41.375919 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.978027 68.144531 67.921875 +util 97.750000 99.000000 95.000000 +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 492 : avg iteration time 46.78 seconds +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 492 : data loading time 0.00 +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 492 : avg forward pass time 12.43 +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 492 : backward pass time 10.18 +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 492 : optimizer step time 1.58 +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 492-------------------- +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:48:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424391 41.532528 41.375923 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.860840 68.144531 67.765625 +util 96.000000 98.000000 93.000000 +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 493 : avg iteration time 46.59 seconds +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 493 : data loading time 0.00 +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 493 : avg forward pass time 12.34 +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 493 : backward pass time 10.15 +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 493 : optimizer step time 1.59 +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 493-------------------- +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:49:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424468 41.532505 41.376038 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.567871 68.144531 67.375000 +util 95.000000 97.000000 92.000000 +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 494 : avg iteration time 46.58 seconds +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 494 : data loading time 0.00 +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 494 : avg forward pass time 12.37 +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 494 : backward pass time 10.13 +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 494 : optimizer step time 1.57 +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 494-------------------- +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:50:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424194 41.531345 41.376053 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.741699 107.863281 107.701172 +util 69.500000 72.000000 66.000000 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 495 : avg iteration time 162.67 seconds +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 495 : data loading time 0.00 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 495 : avg forward pass time 72.64 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 495 : backward pass time 8.47 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 495 : optimizer step time 0.40 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 495-------------------- +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6064 iter count: 1.0 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6064 iter count: 1.0 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6064 iter count: 1.0 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6064iter count: 1.0 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:52:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424231 41.531471 41.376057 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455566 67.871094 67.316406 +util 94.000000 96.000000 91.000000 +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 496 : avg iteration time 48.11 seconds +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 496 : data loading time 0.00 +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 496 : avg forward pass time 13.12 +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 496 : backward pass time 10.14 +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 496 : optimizer step time 1.57 +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 496-------------------- +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:53:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424415 41.532207 41.376091 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484863 67.871094 67.355469 +util 97.000000 99.000000 94.000000 +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 497 : avg iteration time 43.84 seconds +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 497 : data loading time 0.00 +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 497 : avg forward pass time 10.98 +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 497 : backward pass time 10.15 +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 497 : optimizer step time 1.59 +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 497-------------------- +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:54:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424447 41.532227 41.376095 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499512 67.871094 67.375000 +util 91.750000 100.000000 88.000000 +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 498 : avg iteration time 46.69 seconds +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 498 : data loading time 0.00 +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 498 : avg forward pass time 12.42 +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 498 : backward pass time 10.14 +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 498 : optimizer step time 1.57 +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 498-------------------- +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:55:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424529 41.532467 41.376163 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.504395 67.949219 67.355469 +util 89.000000 91.000000 86.000000 +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 499 : avg iteration time 46.62 seconds +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 499 : data loading time 0.00 +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 499 : avg forward pass time 12.39 +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 499 : backward pass time 10.14 +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 499 : optimizer step time 1.56 +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 499-------------------- +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:55:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.424330 41.531635 41.376194 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.814941 108.507812 107.583984 +util 65.250000 66.000000 63.000000 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 500 : avg iteration time 162.51 seconds +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 500 : data loading time 0.00 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 500 : avg forward pass time 72.49 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 500 : backward pass time 8.50 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 500 : optimizer step time 0.42 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 500-------------------- +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5566 iter count: 1.0 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5566 iter count: 1.0 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5566 iter count: 1.0 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5566iter count: 1.0 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 22:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 22:58:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.90 seconds +[May 1, 2026 - 22:58:51 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 500 +[May 1, 2026 - 22:58:51 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000500 +[May 1, 2026 - 22:59:32 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 500 +[May 1, 2026 - 22:59:32 | INFO | fastgen.trainer:validate:358 ] Validation iteration 500 +[May 1, 2026 - 22:59:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 22:59:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.3s +[May 1, 2026 - 22:59:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.294 +[May 1, 2026 - 22:59:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 22:59:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 22:59:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=4.996 +[May 1, 2026 - 22:59:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 22:59:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 22:59:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.754 +[May 1, 2026 - 22:59:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 22:59:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 22:59:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.761 +[May 1, 2026 - 22:59:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 22:59:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 22:59:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.857 +[May 1, 2026 - 22:59:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 23:00:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 23:00:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.511 +[May 1, 2026 - 23:00:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 23:00:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 23:00:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.571 +[May 1, 2026 - 23:00:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 23:00:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 23:00:14 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.682 +[May 1, 2026 - 23:00:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 23:00:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 23:00:19 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.701 +[May 1, 2026 - 23:00:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 23:00:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 23:00:24 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.017 +[May 1, 2026 - 23:00:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 500-------------------- +[May 1, 2026 - 23:00:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 23:00:41 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 500 +[May 1, 2026 - 23:00:41 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.215 (n=10) +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426538 41.540108 41.376301 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.431641 68.789062 68.312500 +util 92.750000 99.000000 86.000000 +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 501 : avg iteration time 173.46 seconds +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 0.00 +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 13.33 +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 10.15 +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 1.59 +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 501-------------------- +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:01:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426587 41.540108 41.376404 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.446289 68.789062 68.332031 +util 95.500000 97.000000 92.000000 +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 46.90 seconds +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 12.46 +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 10.16 +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 1.56 +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 502-------------------- +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:02:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426482 41.539642 41.376419 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.294922 68.769531 68.136719 +util 90.750000 99.000000 87.000000 +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 46.91 seconds +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 12.47 +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 10.22 +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 1.57 +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 503-------------------- +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:03:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426490 41.539642 41.376438 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.412109 68.769531 68.292969 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 49.55 seconds +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 13.82 +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 10.16 +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 1.57 +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 504-------------------- +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:03:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425249 41.534698 41.376415 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.642578 107.609375 +util 61.000000 70.000000 57.000000 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 162.86 seconds +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 72.75 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 8.46 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.40 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 505-------------------- +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5200 iter count: 1.0 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5200 iter count: 1.0 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5200 iter count: 1.0 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5200iter count: 1.0 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:06:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425309 41.534798 41.376438 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.640625 67.667969 67.558594 +util 95.000000 97.000000 92.000000 +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 47.68 seconds +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 12.92 +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 10.15 +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 1.57 +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 506-------------------- +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:07:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425518 41.535553 41.376415 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.806641 67.902344 67.519531 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 46.68 seconds +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 12.40 +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 10.15 +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 1.59 +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 507-------------------- +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:08:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425740 41.536316 41.376438 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.650391 67.667969 67.597656 +util 99.000000 99.000000 99.000000 +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 46.80 seconds +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 12.47 +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 10.13 +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 1.57 +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 508-------------------- +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:08:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425755 41.536293 41.376415 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.796875 67.902344 67.480469 +util 95.250000 97.000000 92.000000 +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 49.73 seconds +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 13.92 +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 10.14 +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 1.58 +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 509-------------------- +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:09:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425384 41.534492 41.376419 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.707031 107.583984 +util 61.500000 64.000000 58.000000 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 159.57 seconds +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 71.15 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 8.42 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.40 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 510-------------------- +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5498 iter count: 1.0 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5498 iter count: 1.0 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5498 iter count: 1.0 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5498iter count: 1.0 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425389 41.534470 41.376457 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.650879 67.656250 67.648438 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 50.67 seconds +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 14.42 +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 10.14 +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 1.57 +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 511-------------------- +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:13:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425538 41.534981 41.376461 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.792480 67.845703 67.636719 +util 98.250000 100.000000 95.000000 +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 46.59 seconds +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 12.37 +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 10.15 +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 1.57 +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 512-------------------- +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:14:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425724 41.535480 41.376461 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.636230 67.669922 67.539062 +util 98.000000 99.000000 95.000000 +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 43.80 seconds +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 10.97 +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 10.13 +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 1.59 +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 513-------------------- +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:14:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425722 41.535473 41.376465 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777832 67.845703 67.578125 +util 99.500000 100.000000 99.000000 +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 44.03 seconds +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 11.05 +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 10.15 +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 1.56 +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 514-------------------- +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:15:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425360 41.533924 41.376526 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.751465 107.779297 107.667969 +util 67.000000 69.000000 64.000000 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 162.17 seconds +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 72.43 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 8.49 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.40 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 515-------------------- +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5850 iter count: 1.0 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5850 iter count: 1.0 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5850 iter count: 1.0 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5850iter count: 1.0 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:18:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425367 41.533939 41.376526 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.850586 67.960938 67.519531 +util 98.000000 99.000000 95.000000 +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 50.40 seconds +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 14.28 +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 10.13 +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 1.56 +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 516-------------------- +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:19:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425530 41.534489 41.376526 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.870117 68.000000 67.480469 +util 96.000000 98.000000 93.000000 +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 43.78 seconds +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 10.94 +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 10.15 +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 1.59 +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 517-------------------- +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:19:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425803 41.535484 41.376503 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.674805 67.746094 67.460938 +util 95.000000 97.000000 92.000000 +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 43.98 seconds +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 11.03 +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 10.14 +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 1.57 +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 518-------------------- +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:20:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425837 41.535465 41.376511 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.708984 67.785156 67.480469 +util 98.750000 100.000000 96.000000 +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 46.65 seconds +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 12.38 +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 10.18 +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 1.56 +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 519-------------------- +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:21:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425544 41.534222 41.376511 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.668457 107.681641 107.628906 +util 68.000000 70.000000 64.000000 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 167.61 seconds +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 75.19 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 8.42 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.39 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 520-------------------- +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5352 iter count: 1.0 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5352 iter count: 1.0 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5352 iter count: 1.0 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5352iter count: 1.0 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:24:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425545 41.534218 41.376518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.582520 67.630859 67.441406 +util 92.250000 100.000000 89.000000 +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 48.27 seconds +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 13.21 +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 10.14 +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 1.58 +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 521-------------------- +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:24:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425697 41.534710 41.376518 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.699707 67.787109 67.441406 +util 98.500000 99.000000 97.000000 +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 49.14 seconds +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.00 +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 13.64 +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 10.13 +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 1.58 +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 522-------------------- +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:25:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425890 41.535469 41.376545 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.719238 67.806641 67.460938 +util 91.750000 99.000000 89.000000 +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 43.85 seconds +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 10.99 +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 10.14 +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 1.56 +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 523-------------------- +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:26:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425938 41.535561 41.376526 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.758301 67.865234 67.441406 +util 95.500000 97.000000 92.000000 +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 524 : avg iteration time 49.29 seconds +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 524 : data loading time 0.00 +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 524 : avg forward pass time 13.72 +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 524 : backward pass time 10.16 +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 524 : optimizer step time 1.57 +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 524-------------------- +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:27:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425553 41.533875 41.376549 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.726562 107.603516 +util 61.500000 64.000000 58.000000 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 525 : avg iteration time 159.35 seconds +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 525 : data loading time 0.00 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 525 : avg forward pass time 71.04 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 525 : backward pass time 8.43 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 525 : optimizer step time 0.39 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 525-------------------- +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5259 iter count: 1.0 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5259 iter count: 1.0 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5259 iter count: 1.0 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5259iter count: 1.0 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:29:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425562 41.533882 41.376526 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.680176 67.728516 67.539062 +util 96.750000 99.000000 93.000000 +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 526 : avg iteration time 44.84 seconds +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 526 : data loading time 0.00 +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 526 : avg forward pass time 11.49 +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 526 : backward pass time 10.13 +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 526 : optimizer step time 1.58 +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 526-------------------- +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:30:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425742 41.534393 41.376526 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729004 67.968750 67.648438 +util 90.500000 99.000000 86.000000 +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 527 : avg iteration time 49.28 seconds +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 527 : data loading time 0.00 +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 527 : avg forward pass time 13.66 +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 527 : backward pass time 10.16 +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 527 : optimizer step time 1.57 +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 527-------------------- +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:31:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425958 41.535152 41.376526 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729004 67.910156 67.667969 +util 91.750000 100.000000 88.000000 +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 528 : avg iteration time 43.81 seconds +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 528 : data loading time 0.00 +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 528 : avg forward pass time 10.97 +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 528 : backward pass time 10.16 +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 528 : optimizer step time 1.58 +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 528-------------------- +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:32:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425993 41.535179 41.376526 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777832 67.865234 67.519531 +util 91.750000 94.000000 88.000000 +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 529 : avg iteration time 49.44 seconds +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 529 : data loading time 0.00 +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 529 : avg forward pass time 13.79 +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 529 : backward pass time 10.14 +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 529 : optimizer step time 1.57 +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 529-------------------- +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:33:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:35:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425691 41.533779 41.376621 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.726562 107.544922 +util 62.750000 65.000000 59.000000 +[May 1, 2026 - 23:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 530 : avg iteration time 159.43 seconds +[May 1, 2026 - 23:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 530 : data loading time 0.00 +[May 1, 2026 - 23:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 530 : avg forward pass time 71.11 +[May 1, 2026 - 23:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 530 : backward pass time 8.44 +[May 1, 2026 - 23:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 530 : optimizer step time 0.39 +[May 1, 2026 - 23:35:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 530-------------------- +[May 1, 2026 - 23:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5967 iter count: 1.0 +[May 1, 2026 - 23:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5967 iter count: 1.0 +[May 1, 2026 - 23:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5967 iter count: 1.0 +[May 1, 2026 - 23:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5967iter count: 1.0 +[May 1, 2026 - 23:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 23:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:35:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425696 41.533779 41.376625 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.602051 67.689453 67.343750 +util 98.000000 99.000000 95.000000 +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 531 : avg iteration time 50.65 seconds +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 531 : data loading time 0.00 +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 531 : avg forward pass time 14.40 +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 531 : backward pass time 10.13 +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 531 : optimizer step time 1.56 +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 531-------------------- +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:36:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425827 41.534294 41.376625 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.645996 67.669922 67.578125 +util 96.500000 98.000000 93.000000 +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 532 : avg iteration time 49.18 seconds +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 532 : data loading time 0.00 +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 532 : avg forward pass time 13.65 +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 532 : backward pass time 10.14 +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 532 : optimizer step time 1.57 +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 532-------------------- +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426150 41.535580 41.376629 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.602051 67.669922 67.402344 +util 91.750000 94.000000 88.000000 +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 533 : avg iteration time 46.54 seconds +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 533 : data loading time 0.00 +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 533 : avg forward pass time 12.34 +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 533 : backward pass time 10.15 +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 533 : optimizer step time 1.57 +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 533-------------------- +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426216 41.535809 41.376640 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.680176 67.767578 67.421875 +util 92.500000 94.000000 89.000000 +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 534 : avg iteration time 46.41 seconds +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 534 : data loading time 0.00 +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 534 : avg forward pass time 12.24 +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 534 : backward pass time 10.22 +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 534 : optimizer step time 1.56 +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 534-------------------- +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:38:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425867 41.534187 41.376648 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.658691 107.662109 107.648438 +util 62.750000 65.000000 59.000000 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 535 : avg iteration time 159.61 seconds +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 535 : data loading time 0.00 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 535 : avg forward pass time 71.15 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 535 : backward pass time 8.47 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 535 : optimizer step time 0.39 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 535-------------------- +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5557 iter count: 1.0 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5557 iter count: 1.0 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5557 iter count: 1.0 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5557iter count: 1.0 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:41:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425868 41.534187 41.376671 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.674805 67.726562 67.519531 +util 91.500000 99.000000 88.000000 +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 536 : avg iteration time 47.54 seconds +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 536 : data loading time 0.00 +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 536 : avg forward pass time 12.83 +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 536 : backward pass time 10.15 +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 536 : optimizer step time 1.57 +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 536-------------------- +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:42:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426035 41.534733 41.376671 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777344 67.863281 67.519531 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 537 : avg iteration time 49.34 seconds +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 537 : data loading time 0.00 +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 537 : avg forward pass time 13.75 +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 537 : backward pass time 10.13 +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 537 : optimizer step time 1.57 +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 537-------------------- +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:43:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426224 41.535480 41.376667 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.787109 67.882812 67.500000 +util 90.750000 99.000000 87.000000 +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 538 : avg iteration time 46.44 seconds +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 538 : data loading time 0.00 +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 538 : avg forward pass time 12.29 +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 538 : backward pass time 10.15 +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 538 : optimizer step time 1.57 +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 538-------------------- +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:43:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426357 41.535984 41.376667 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.806641 67.902344 67.519531 +util 89.000000 90.000000 87.000000 +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 539 : avg iteration time 49.24 seconds +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 539 : data loading time 0.00 +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 539 : avg forward pass time 13.68 +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 539 : backward pass time 10.13 +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 539 : optimizer step time 1.59 +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 539-------------------- +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:44:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426064 41.534821 41.376682 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.726562 107.564453 +util 67.000000 70.000000 63.000000 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 540 : avg iteration time 161.90 seconds +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 540 : data loading time 0.00 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 540 : avg forward pass time 72.34 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 540 : backward pass time 8.45 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 540 : optimizer step time 0.40 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 540-------------------- +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5830 iter count: 1.0 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5830 iter count: 1.0 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5830 iter count: 1.0 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5830iter count: 1.0 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:47:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425867 41.533890 41.376682 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358398 67.480469 67.316406 +util 99.000000 100.000000 98.000000 +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 541 : avg iteration time 50.23 seconds +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 541 : data loading time 0.00 +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 541 : avg forward pass time 14.20 +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 541 : backward pass time 10.13 +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 541 : optimizer step time 1.55 +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 541-------------------- +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:48:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425889 41.533894 41.376705 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.207031 67.402344 67.140625 +util 97.500000 99.000000 94.000000 +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 542 : avg iteration time 46.35 seconds +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 542 : data loading time 0.00 +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 542 : avg forward pass time 12.26 +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 542 : backward pass time 10.13 +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 542 : optimizer step time 1.58 +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 542-------------------- +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:49:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425890 41.533916 41.376705 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402344 67.421875 67.394531 +util 99.250000 100.000000 99.000000 +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 543 : avg iteration time 46.65 seconds +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 543 : data loading time 0.00 +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 543 : avg forward pass time 12.40 +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 543 : backward pass time 10.13 +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 543 : optimizer step time 1.57 +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 543-------------------- +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:49:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425872 41.533909 41.376686 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.226562 67.480469 67.140625 +util 97.500000 99.000000 94.000000 +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 544 : avg iteration time 49.34 seconds +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 544 : data loading time 0.00 +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 544 : avg forward pass time 13.73 +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 544 : backward pass time 10.15 +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 544 : optimizer step time 1.58 +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 544-------------------- +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:50:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426040 41.534393 41.376717 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.765625 107.564453 +util 68.750000 71.000000 65.000000 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 545 : avg iteration time 162.25 seconds +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 545 : data loading time 0.00 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 545 : avg forward pass time 72.50 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 545 : backward pass time 8.40 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 545 : optimizer step time 0.41 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 545-------------------- +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6143 iter count: 1.0 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6143 iter count: 1.0 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6143 iter count: 1.0 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6143iter count: 1.0 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:53:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425905 41.533875 41.376717 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.650879 67.728516 67.421875 +util 97.500000 99.000000 94.000000 +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 546 : avg iteration time 47.75 seconds +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 546 : data loading time 0.00 +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 546 : avg forward pass time 12.93 +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 546 : backward pass time 10.17 +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 546 : optimizer step time 1.56 +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 546-------------------- +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:54:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425919 41.533897 41.376747 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.797363 67.865234 67.597656 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 547 : avg iteration time 43.83 seconds +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 547 : data loading time 0.00 +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 547 : avg forward pass time 10.99 +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 547 : backward pass time 10.13 +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 547 : optimizer step time 1.57 +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 547-------------------- +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:54:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425913 41.533897 41.376701 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.592285 67.630859 67.480469 +util 91.750000 94.000000 88.000000 +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 548 : avg iteration time 49.31 seconds +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 548 : data loading time 0.00 +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 548 : avg forward pass time 13.65 +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 548 : backward pass time 10.15 +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 548 : optimizer step time 1.58 +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 548-------------------- +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:55:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425910 41.533905 41.376705 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729004 67.845703 67.382812 +util 95.750000 98.000000 92.000000 +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 549 : avg iteration time 49.44 seconds +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 549 : data loading time 0.00 +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 549 : avg forward pass time 13.70 +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 549 : backward pass time 10.23 +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 549 : optimizer step time 1.57 +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 549-------------------- +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:56:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425928 41.533905 41.376709 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.707031 107.603516 +util 62.500000 71.000000 58.000000 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 550 : avg iteration time 156.53 seconds +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 550 : data loading time 0.00 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 550 : avg forward pass time 69.66 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 550 : backward pass time 8.40 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 550 : optimizer step time 0.39 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 550-------------------- +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5220 iter count: 1.0 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5220 iter count: 1.0 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5220 iter count: 1.0 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5220iter count: 1.0 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:59:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425926 41.533905 41.376732 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.863281 67.460938 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 551 : avg iteration time 44.92 seconds +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 551 : data loading time 0.00 +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 551 : avg forward pass time 11.53 +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 551 : backward pass time 10.15 +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 551 : optimizer step time 1.58 +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 551-------------------- +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 23:59:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425930 41.533882 41.376724 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.743164 67.843750 67.441406 +util 98.250000 99.000000 96.000000 +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 552 : avg iteration time 46.61 seconds +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 552 : data loading time 0.00 +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 552 : avg forward pass time 12.30 +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 552 : backward pass time 10.14 +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 552 : optimizer step time 1.59 +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 552-------------------- +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:00:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425936 41.533882 41.376724 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.733398 67.843750 67.402344 +util 93.250000 95.000000 90.000000 +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 553 : avg iteration time 46.52 seconds +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 553 : data loading time 0.00 +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 553 : avg forward pass time 12.34 +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 553 : backward pass time 10.13 +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 553 : optimizer step time 1.56 +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 553-------------------- +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:01:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426146 41.534634 41.376732 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.843750 67.500000 +util 95.750000 98.000000 92.000000 +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 554 : avg iteration time 46.43 seconds +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 554 : data loading time 0.00 +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 554 : avg forward pass time 12.29 +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 554 : backward pass time 10.14 +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 554 : optimizer step time 1.57 +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 554-------------------- +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:02:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426043 41.534027 41.376751 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.687500 107.583984 +util 61.500000 64.000000 57.000000 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 555 : avg iteration time 162.39 seconds +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 555 : data loading time 0.00 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 555 : avg forward pass time 72.50 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 555 : backward pass time 8.45 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 555 : optimizer step time 0.41 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 555-------------------- +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5371 iter count: 1.0 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5371 iter count: 1.0 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5371 iter count: 1.0 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5371iter count: 1.0 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:04:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426052 41.534031 41.376751 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.812500 67.884766 67.597656 +util 97.750000 100.000000 94.000000 +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 556 : avg iteration time 47.65 seconds +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 556 : data loading time 0.00 +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 556 : avg forward pass time 12.87 +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 556 : backward pass time 10.17 +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 556 : optimizer step time 1.57 +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 556-------------------- +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:05:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426051 41.534050 41.376751 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.700195 67.728516 67.617188 +util 99.250000 100.000000 99.000000 +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 557 : avg iteration time 49.25 seconds +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 557 : data loading time 0.00 +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 557 : avg forward pass time 13.69 +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 557 : backward pass time 10.14 +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 557 : optimizer step time 1.57 +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 557-------------------- +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:06:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426111 41.534306 41.376751 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.705078 67.748047 67.578125 +util 98.000000 99.000000 95.000000 +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 558 : avg iteration time 43.77 seconds +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 558 : data loading time 0.00 +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 558 : avg forward pass time 10.93 +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 558 : backward pass time 10.17 +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 558 : optimizer step time 1.58 +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 558-------------------- +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:07:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426246 41.534840 41.376751 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.651367 67.669922 67.597656 +util 97.250000 99.000000 93.000000 +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 559 : avg iteration time 46.58 seconds +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 559 : data loading time 0.00 +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 559 : avg forward pass time 12.37 +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 559 : backward pass time 10.14 +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 559 : optimizer step time 1.56 +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 559-------------------- +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:08:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426139 41.534367 41.376751 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.628906 107.525391 +util 60.750000 69.000000 56.000000 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 560 : avg iteration time 159.10 seconds +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 560 : data loading time 0.00 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 560 : avg forward pass time 70.95 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 560 : backward pass time 8.41 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 560 : optimizer step time 0.40 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 560-------------------- +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5361 iter count: 1.0 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5361 iter count: 1.0 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5361 iter count: 1.0 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5361iter count: 1.0 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:10:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426132 41.534351 41.376762 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.636230 67.689453 67.480469 +util 99.000000 100.000000 97.000000 +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 561 : avg iteration time 47.55 seconds +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 561 : data loading time 0.00 +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 561 : avg forward pass time 12.83 +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 561 : backward pass time 10.16 +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 561 : optimizer step time 1.58 +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 561-------------------- +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:11:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426131 41.534351 41.376759 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.606934 67.650391 67.480469 +util 96.250000 98.000000 92.000000 +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 562 : avg iteration time 49.15 seconds +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 562 : data loading time 0.00 +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 562 : avg forward pass time 13.62 +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 562 : backward pass time 10.19 +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 562 : optimizer step time 1.56 +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 562-------------------- +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:12:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426152 41.534374 41.376755 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.602051 67.650391 67.460938 +util 91.250000 93.000000 87.000000 +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 563 : avg iteration time 43.74 seconds +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 563 : data loading time 0.00 +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 563 : avg forward pass time 10.93 +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 563 : backward pass time 10.14 +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 563 : optimizer step time 1.58 +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 563-------------------- +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:13:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426271 41.534874 41.376755 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.606934 67.650391 67.480469 +util 91.250000 93.000000 87.000000 +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 564 : avg iteration time 46.40 seconds +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 564 : data loading time 0.00 +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 564 : avg forward pass time 12.28 +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 564 : backward pass time 10.13 +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 564 : optimizer step time 1.58 +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 564-------------------- +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:13:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426149 41.534355 41.376759 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.746094 107.603516 +util 59.500000 62.000000 55.000000 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 565 : avg iteration time 159.60 seconds +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 565 : data loading time 0.00 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 565 : avg forward pass time 71.18 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 565 : backward pass time 8.46 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 565 : optimizer step time 0.39 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 565-------------------- +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5093 iter count: 1.0 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5093 iter count: 1.0 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5093 iter count: 1.0 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5093iter count: 1.0 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:16:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426163 41.534382 41.376785 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.630859 67.687500 67.460938 +util 95.000000 97.000000 91.000000 +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 566 : avg iteration time 47.65 seconds +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 566 : data loading time 0.00 +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 566 : avg forward pass time 12.91 +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 566 : backward pass time 10.14 +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 566 : optimizer step time 1.56 +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 566-------------------- +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:17:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426215 41.534607 41.376770 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.831055 67.863281 67.734375 +util 98.750000 100.000000 97.000000 +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 567 : avg iteration time 43.87 seconds +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 567 : data loading time 0.00 +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 567 : avg forward pass time 11.00 +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 567 : backward pass time 10.13 +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 567 : optimizer step time 1.57 +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 567-------------------- +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:18:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426229 41.534622 41.376770 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.665039 67.707031 67.539062 +util 93.500000 100.000000 90.000000 +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 568 : avg iteration time 46.47 seconds +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 568 : data loading time 0.00 +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 568 : avg forward pass time 12.32 +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 568 : backward pass time 10.13 +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 568 : optimizer step time 1.56 +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 568-------------------- +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:18:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426234 41.534630 41.376770 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.843750 67.539062 +util 98.250000 100.000000 95.000000 +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 569 : avg iteration time 46.51 seconds +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 569 : data loading time 0.00 +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 569 : avg forward pass time 12.34 +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 569 : backward pass time 10.13 +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 569 : optimizer step time 1.56 +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 569-------------------- +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:19:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426057 41.533924 41.376774 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.746094 107.525391 +util 69.000000 71.000000 65.000000 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 570 : avg iteration time 162.21 seconds +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 570 : data loading time 0.00 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 570 : avg forward pass time 72.44 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 570 : backward pass time 8.49 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 570 : optimizer step time 0.39 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 570-------------------- +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5898 iter count: 1.0 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5898 iter count: 1.0 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5898 iter count: 1.0 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5898iter count: 1.0 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:22:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426070 41.533947 41.376785 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.655273 67.687500 67.558594 +util 90.500000 93.000000 86.000000 +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 571 : avg iteration time 50.29 seconds +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 571 : data loading time 0.00 +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 571 : avg forward pass time 14.21 +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 571 : backward pass time 10.15 +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 571 : optimizer step time 1.57 +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 571-------------------- +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:23:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426077 41.533939 41.376774 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.625977 67.648438 67.558594 +util 95.250000 98.000000 91.000000 +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 572 : avg iteration time 43.85 seconds +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 572 : data loading time 0.00 +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 572 : avg forward pass time 10.99 +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 572 : backward pass time 10.16 +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 572 : optimizer step time 1.57 +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 572-------------------- +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:23:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426083 41.533936 41.376797 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.824219 67.558594 +util 97.250000 99.000000 93.000000 +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 573 : avg iteration time 46.43 seconds +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 573 : data loading time 0.00 +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 573 : avg forward pass time 12.27 +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 573 : backward pass time 10.15 +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 573 : optimizer step time 1.57 +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 573-------------------- +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:24:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426080 41.533936 41.376774 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.625977 67.648438 67.558594 +util 91.500000 94.000000 87.000000 +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 574 : avg iteration time 43.73 seconds +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 574 : data loading time 0.00 +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 574 : avg forward pass time 10.91 +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 574 : backward pass time 10.18 +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 574 : optimizer step time 1.58 +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 574-------------------- +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:25:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425900 41.533180 41.376801 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.687500 107.642578 +util 60.000000 62.000000 56.000000 +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 575 : avg iteration time 164.80 seconds +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 575 : data loading time 0.00 +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 575 : avg forward pass time 73.77 +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 575 : backward pass time 8.43 +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 575 : optimizer step time 0.40 +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 575-------------------- +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5508 iter count: 1.0 +[May 2, 2026 - 00:28:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5508 iter count: 1.0 +[May 2, 2026 - 00:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5508 iter count: 1.0 +[May 2, 2026 - 00:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5508iter count: 1.0 +[May 2, 2026 - 00:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 2, 2026 - 00:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:28:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425894 41.533180 41.376778 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.889648 67.921875 67.792969 +util 97.500000 99.000000 94.000000 +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 576 : avg iteration time 50.28 seconds +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 576 : data loading time 0.00 +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 576 : avg forward pass time 14.17 +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 576 : backward pass time 10.15 +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 576 : optimizer step time 1.57 +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 576-------------------- +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:28:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426083 41.533958 41.376778 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.928711 67.949219 67.921875 +util 98.000000 99.000000 95.000000 +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 577 : avg iteration time 43.82 seconds +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 577 : data loading time 0.00 +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 577 : avg forward pass time 10.99 +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 577 : backward pass time 10.13 +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 577 : optimizer step time 1.58 +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 577-------------------- +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:29:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426086 41.533943 41.376789 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 67.753906 67.746094 +util 90.000000 99.000000 86.000000 +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 578 : avg iteration time 46.44 seconds +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 578 : data loading time 0.00 +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 578 : avg forward pass time 12.29 +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 578 : backward pass time 10.14 +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 578 : optimizer step time 1.57 +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 578-------------------- +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:30:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426183 41.534180 41.376781 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.855469 67.882812 67.773438 +util 93.250000 99.000000 90.000000 +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 579 : avg iteration time 46.46 seconds +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 579 : data loading time 0.00 +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 579 : avg forward pass time 12.29 +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 579 : backward pass time 10.14 +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 579 : optimizer step time 1.58 +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 579-------------------- +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 2, 2026 - 00:31:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:31:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425903 41.533062 41.376781 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.561035 107.667969 107.525391 +util 67.750000 70.000000 63.000000 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 580 : avg iteration time 162.09 seconds +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 580 : data loading time 0.00 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 580 : avg forward pass time 72.42 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 580 : backward pass time 8.44 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 580 : optimizer step time 0.40 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 580-------------------- +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5527 iter count: 1.0 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5527 iter count: 1.0 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5527 iter count: 1.0 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5527iter count: 1.0 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:33:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425908 41.533085 41.376797 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.699219 67.824219 67.324219 +util 89.750000 92.000000 85.000000 +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 581 : avg iteration time 47.90 seconds +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 581 : data loading time 0.00 +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 581 : avg forward pass time 13.03 +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 581 : backward pass time 10.12 +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 581 : optimizer step time 1.56 +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 581-------------------- +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:34:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426030 41.533566 41.376789 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.796875 67.902344 67.480469 +util 90.500000 93.000000 86.000000 +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 582 : avg iteration time 43.80 seconds +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 582 : data loading time 0.00 +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 582 : avg forward pass time 10.95 +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 582 : backward pass time 10.16 +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 582 : optimizer step time 1.58 +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 582-------------------- +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:35:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426232 41.534317 41.376812 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 91.500000 99.000000 88.000000 +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 583 : avg iteration time 46.56 seconds +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 583 : data loading time 0.00 +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 583 : avg forward pass time 12.37 +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 583 : backward pass time 10.11 +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 583 : optimizer step time 1.56 +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 583-------------------- +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:36:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426247 41.534340 41.376797 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.882812 67.421875 +util 91.000000 99.000000 87.000000 +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 584 : avg iteration time 46.60 seconds +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 584 : data loading time 0.00 +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 584 : avg forward pass time 12.37 +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 584 : backward pass time 10.13 +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 584 : optimizer step time 1.57 +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 584-------------------- +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:37:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425877 41.532841 41.376820 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.726562 107.544922 +util 67.000000 69.000000 63.000000 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 585 : avg iteration time 165.12 seconds +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 585 : data loading time 0.00 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 585 : avg forward pass time 73.94 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 585 : backward pass time 8.44 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 585 : optimizer step time 0.39 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 585-------------------- +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5908 iter count: 1.0 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5908 iter count: 1.0 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5908 iter count: 1.0 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5908iter count: 1.0 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:39:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.425871 41.532848 41.376801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.763672 67.865234 67.460938 +util 92.750000 99.000000 89.000000 +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 586 : avg iteration time 45.09 seconds +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 586 : data loading time 0.00 +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 586 : avg forward pass time 11.60 +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 586 : backward pass time 10.10 +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 586 : optimizer step time 1.56 +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 586-------------------- +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:40:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426003 41.533360 41.376797 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.602539 67.650391 67.460938 +util 98.500000 99.000000 97.000000 +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 587 : avg iteration time 43.75 seconds +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 587 : data loading time 0.00 +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 587 : avg forward pass time 10.94 +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 587 : backward pass time 10.15 +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 587 : optimizer step time 1.57 +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 587-------------------- +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:41:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426198 41.534119 41.376820 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.641602 67.708984 67.441406 +util 91.750000 99.000000 88.000000 +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 588 : avg iteration time 46.65 seconds +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 588 : data loading time 0.00 +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 588 : avg forward pass time 12.33 +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 588 : backward pass time 10.25 +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 588 : optimizer step time 1.60 +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 588-------------------- +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:42:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426198 41.534119 41.376842 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.602539 67.689453 67.343750 +util 92.500000 99.000000 89.000000 +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 589 : avg iteration time 46.60 seconds +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 589 : data loading time 0.00 +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 589 : avg forward pass time 12.30 +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 589 : backward pass time 10.15 +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 589 : optimizer step time 1.59 +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 589-------------------- +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:42:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426256 41.534367 41.376804 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.648438 107.544922 +util 60.000000 62.000000 56.000000 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 590 : avg iteration time 159.43 seconds +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 590 : data loading time 0.00 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 590 : avg forward pass time 71.10 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 590 : backward pass time 8.42 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 590 : optimizer step time 0.40 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 590-------------------- +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5591 iter count: 1.0 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5591 iter count: 1.0 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5591 iter count: 1.0 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5591iter count: 1.0 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:45:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426025 41.533455 41.376808 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.739258 67.826172 67.480469 +util 96.500000 99.000000 92.000000 +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 591 : avg iteration time 48.06 seconds +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 591 : data loading time 0.00 +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 591 : avg forward pass time 13.09 +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 591 : backward pass time 10.14 +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 591 : optimizer step time 1.59 +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 591-------------------- +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:46:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426045 41.533489 41.376801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.675781 67.767578 67.402344 +util 94.000000 99.000000 91.000000 +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 592 : avg iteration time 49.53 seconds +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 592 : data loading time 0.00 +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 592 : avg forward pass time 13.81 +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 592 : backward pass time 10.18 +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 592 : optimizer step time 1.57 +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 592-------------------- +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:47:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426100 41.533695 41.376801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.602539 67.669922 67.402344 +util 96.250000 98.000000 92.000000 +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 593 : avg iteration time 46.66 seconds +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 593 : data loading time 0.00 +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 593 : avg forward pass time 12.40 +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 593 : backward pass time 10.16 +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 593 : optimizer step time 1.58 +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 593-------------------- +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:47:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426097 41.533695 41.376801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.734375 67.845703 67.402344 +util 98.250000 99.000000 96.000000 +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 594 : avg iteration time 44.04 seconds +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 594 : data loading time 0.00 +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 594 : avg forward pass time 11.08 +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 594 : backward pass time 10.16 +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 594 : optimizer step time 1.58 +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 594-------------------- +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:48:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426316 41.534473 41.376801 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.663574 107.785156 107.623047 +util 59.000000 63.000000 54.000000 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 595 : avg iteration time 162.95 seconds +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 595 : data loading time 0.00 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 595 : avg forward pass time 72.81 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 595 : backward pass time 8.43 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 595 : optimizer step time 0.46 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 595-------------------- +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5537 iter count: 1.0 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5537 iter count: 1.0 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5537 iter count: 1.0 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5537iter count: 1.0 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:51:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426219 41.534016 41.376820 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.843750 67.500000 +util 97.500000 99.000000 94.000000 +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 596 : avg iteration time 45.23 seconds +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 596 : data loading time 0.00 +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 596 : avg forward pass time 11.66 +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 596 : backward pass time 10.15 +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 596 : optimizer step time 1.60 +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 596-------------------- +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:52:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426166 41.533840 41.376801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.743164 67.843750 67.441406 +util 98.750000 100.000000 97.000000 +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 597 : avg iteration time 44.03 seconds +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 597 : data loading time 0.00 +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 597 : avg forward pass time 11.07 +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 597 : backward pass time 10.16 +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 597 : optimizer step time 1.57 +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 597-------------------- +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:52:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426252 41.534069 41.376801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.616211 67.667969 67.460938 +util 93.750000 96.000000 89.000000 +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 598 : avg iteration time 49.51 seconds +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 598 : data loading time 0.00 +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 598 : avg forward pass time 13.79 +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 598 : backward pass time 10.16 +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 598 : optimizer step time 1.57 +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 598-------------------- +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:53:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.426257 41.534069 41.376808 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.635742 67.707031 67.421875 +util 97.250000 99.000000 93.000000 +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 599 : avg iteration time 49.65 seconds +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 599 : data loading time 0.00 +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 599 : avg forward pass time 13.87 +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 599 : backward pass time 10.16 +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 599 : optimizer step time 1.58 +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 599-------------------- +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 2, 2026 - 00:54:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 2, 2026 - 00:54:27 | INFO | fastgen.trainer:run:225 ] Training complete. +[May 2, 2026 - 00:54:27 | INFO | fastgen.trainer:validate:358 ] Validation iteration 600 +[May 2, 2026 - 00:54:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:54:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:54:32 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.457 +[May 2, 2026 - 00:54:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:54:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:54:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.661 +[May 2, 2026 - 00:54:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:54:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:54:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.093 +[May 2, 2026 - 00:54:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:54:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:54:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.508 +[May 2, 2026 - 00:54:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:54:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:54:54 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=3.622 +[May 2, 2026 - 00:54:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:54:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:54:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.489 +[May 2, 2026 - 00:54:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:55:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.327 +[May 2, 2026 - 00:55:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:55:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:55:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.511 +[May 2, 2026 - 00:55:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:55:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:55:14 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.557 +[May 2, 2026 - 00:55:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 2, 2026 - 00:55:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 2, 2026 - 00:55:19 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.965 +[May 2, 2026 - 00:55:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 600-------------------- +[May 2, 2026 - 00:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 2, 2026 - 00:55:36 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 600 +[May 2, 2026 - 00:55:36 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.119 (n=10) +[May 2, 2026 - 00:55:36 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 600 +[May 2, 2026 - 00:55:36 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600 +[May 2, 2026 - 00:56:15 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 600 +[May 2, 2026 - 00:56:16 | INFO | fastgen.trainer:run:232 ] Taking a 10 sec nap and exiting training. +[May 2, 2026 - 00:56:26 | INFO | fastgen.utils.distributed:clean_up:88 ] Distributed clean up. +[May 2, 2026 - 00:56:27 | INFO | __main__::49 ] Training finished. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/requirements.txt b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/wandb-metadata.json b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1078848dfd3f64e3a051690f7c40c01997d76d9d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/wandb-metadata.json @@ -0,0 +1,70 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-01T03:59:36.507877Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769_no_reward.py", + "-", + "trainer.resume=False", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769_no_reward", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "f5897815a4bdcc8f11212b24f367a6839a5409ca" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1159903330304" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "ybjk1atzxhtxhspjhbgv689r015pk5wo" +} \ No newline at end of file diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/wandb-summary.json b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..18363052db35728f9f074f0e71fe3ec8eee3158d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/files/wandb-summary.json @@ -0,0 +1 @@ +{"val0/sync_c_sample_6":6.327133655548096,"net/local_total_params":[355555624,355276072,355276072,355276072],"_wandb":{"runtime":43008},"net/total_params":1421383840,"train/vsd_loss_unweighted":0.5537109375,"train/gan_loss_gen":0,"val0/sync_c_sample_3":4.508382797241211,"optimizer/lr_net":2e-06,"_step":600,"_runtime":43008.952151087,"profiler/data_loading_time":0.0009641144424676895,"train_media/data/real":{"size":190461,"path":"media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4","_type":"video-file","sha256":"2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae"},"val0/sync_c_mean":5.1189569473266605,"optimizer/grad_norm (model_key net)":0,"train/vsd_loss_weighted":0.5537109375,"optimizer/non_finite_grads_count (model_key net)":0,"fake_score/total_params":1421383840,"profiler/avg_forward_pass_time":13.873143295757473,"optimizer/iteration":600,"train/gan_loss_disc":0,"profiler/backward_pass_time":10.159759571775794,"train/total_loss":0.0093536376953125,"val0/sync_c_sample_2":5.093072414398193,"model/total_params":17136849376,"train/fake_score_loss":0.0093536376953125,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"model/local_trainable_params":[711111248,710552144,710552144,710552144],"profiler/optimizer_step_time":1.5770540162920952,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"val0/sync_c_sample_1":5.660599708557129,"net/trainable_params":1421383840,"val0/sync_c_sample_7":4.510529518127441,"val0/sync_c_sample_8":5.5569071769714355,"val0/sync_c_sample_0":6.456800937652588,"_timestamp":1.7776509366331735e+09,"train/vsd_loss":0.5537109375,"model/local_total_params":[4285560952,4283762808,4283762808,4283762808],"val0/total_loss":0,"val0/generated":{"captions":["sync_c=6.457","sync_c=5.661","sync_c=5.093","sync_c=4.508","sync_c=3.622","sync_c=6.489","sync_c=6.327","sync_c=4.511","sync_c=5.557","sync_c=2.965"],"_type":"videos","count":10,"videos":[{"size":217355,"path":"media/videos/val0/generated_600_66e24e5fe2e1c0fe4f8a.mp4","caption":"sync_c=6.457","_type":"video-file","sha256":"66e24e5fe2e1c0fe4f8ac1ed74cf66f343a6dd2bcfa9ca17e07e81de2d8ba022"},{"caption":"sync_c=5.661","_type":"video-file","sha256":"c634261dd6b7f89f930efed13fb0188a0f9139e4c89ecd5e56befd0ca715abab","size":218831,"path":"media/videos/val0/generated_600_c634261dd6b7f89f930e.mp4"},{"caption":"sync_c=5.093","_type":"video-file","sha256":"86754b3d2902e460dd2ce4f16fd8f4e6609287a1f8f1352e19e9a9a3d801399b","size":211679,"path":"media/videos/val0/generated_600_86754b3d2902e460dd2c.mp4"},{"caption":"sync_c=4.508","_type":"video-file","sha256":"1daf3ffd78eaedf00b9380ee4b516b2b722a5689e49840aee293d26eb8c96d57","size":299976,"path":"media/videos/val0/generated_600_1daf3ffd78eaedf00b93.mp4"},{"sha256":"dc0f8b7980bb0406ab08085cde20ebc923f209aff600cc097dbf6637b11ecb71","size":262007,"path":"media/videos/val0/generated_600_dc0f8b7980bb0406ab08.mp4","caption":"sync_c=3.622","_type":"video-file"},{"sha256":"6213360a5c574ef5b5da2f23aa1bfd8d09e250309e529b2ddbe9d4f0cf8e83b4","size":188305,"path":"media/videos/val0/generated_600_6213360a5c574ef5b5da.mp4","caption":"sync_c=6.489","_type":"video-file"},{"_type":"video-file","sha256":"30ebc58e2c2585a478e2f07f953d3eea5395425ab91c63059b1259f80dbda6a7","size":307007,"path":"media/videos/val0/generated_600_30ebc58e2c2585a478e2.mp4","caption":"sync_c=6.327"},{"path":"media/videos/val0/generated_600_13dd880e1be3afbc4305.mp4","caption":"sync_c=4.511","_type":"video-file","sha256":"13dd880e1be3afbc4305b5bb6a8c99b734d6caaefe1e76344bebdb72818f1df7","size":201215},{"caption":"sync_c=5.557","_type":"video-file","sha256":"c7bfada6d2a58a176cebf022886cd2274d7f8868f2c97217b7c145efda19e73d","size":244270,"path":"media/videos/val0/generated_600_c7bfada6d2a58a176ceb.mp4"},{"size":250738,"path":"media/videos/val0/generated_600_5233db7fa378d994b2bf.mp4","caption":"sync_c=2.965","_type":"video-file","sha256":"5233db7fa378d994b2bf6239832f9a05bf530070379eafc19084f94c3eeebe67"}]},"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"optimizer/lr_fake_score":2e-06,"val0/sync_c_sample_4":3.621853828430176,"val0/sync_c_sample_9":2.9652099609375,"val0/sync_c_sample_5":6.489079475402832,"fake_score/trainable_params":1421383840,"profiler/avg_iteration_time":49.64675331115723,"val0/reconstructed":{"_type":"videos","count":10,"videos":[{"_type":"video-file","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028,"path":"media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4"},{"size":170922,"path":"media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4","_type":"video-file","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214"},{"_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8","size":173236,"path":"media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4"},{"_type":"video-file","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372,"path":"media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4"},{"size":190004,"path":"media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4","_type":"video-file","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117"},{"_type":"video-file","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630,"path":"media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4"},{"sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986,"path":"media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4","_type":"video-file"},{"_type":"video-file","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516,"path":"media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4"},{"size":214001,"path":"media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4","_type":"video-file","sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac"},{"size":227490,"path":"media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4","_type":"video-file","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273"}],"captions":false},"model/trainable_params":2842767680,"train_media/student/generation":{"sha256":"54d7e0b7b8631130c0130dc593aca13885eead1f40bb955833b5027567d55eaa","size":215988,"path":"media/videos/train_media/student/generation_500_54d7e0b7b8631130c013.mp4","_type":"video-file"}} \ No newline at end of file diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug-core.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..edaac81e325ee4e4b7cbaa23a7e177388f09bcd0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-01T12:59:37.020398897+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphxk4hcug/port-2521108.txt","pid":2521108,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-01T12:59:37.022006165+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2521108-2540039-870803298/socket","Net":"unix"}} +{"time":"2026-05-01T12:59:37.022973827+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2521108} +{"time":"2026-05-01T12:59:37.069826167+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-01T12:59:37.120056299+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lw2bm1l6","id":"1(@)"} +{"time":"2026-05-01T12:59:37.512208376+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lw2bm1l6","id":"1(@)"} +{"time":"2026-05-01T12:59:43.966189424+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"5580ew5lpccs"} +{"time":"2026-05-02T00:56:27.117756095+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-02T00:56:27.119291987+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-02T00:56:27.11935915+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-02T00:56:27.120863191+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-02T00:56:27.124669612+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2521108-2540039-870803298/socket","Net":"unix"}} +{"time":"2026-05-02T00:56:28.767041133+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-02T00:56:28.767067948+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-02T00:56:28.767079064+09:00","level":"INFO","msg":"server is closed"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug-internal.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6f1bedc5866126a6b48a9dbf6971163e37861944 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-01T12:59:37.121302481+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-01T12:59:37.510068982+09:00","level":"INFO","msg":"stream: created new stream","id":"lw2bm1l6"} +{"time":"2026-05-01T12:59:37.511011376+09:00","level":"INFO","msg":"handler: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-01T12:59:37.512201036+09:00","level":"INFO","msg":"stream: started","id":"lw2bm1l6"} +{"time":"2026-05-01T12:59:37.512262583+09:00","level":"INFO","msg":"writer: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-01T12:59:37.512289211+09:00","level":"INFO","msg":"sender: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-02T00:56:27.124877654+09:00","level":"INFO","msg":"stream: closing","id":"lw2bm1l6"} +{"time":"2026-05-02T00:56:28.442505939+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-02T00:56:28.761865906+09:00","level":"INFO","msg":"handler: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-02T00:56:28.763250616+09:00","level":"INFO","msg":"sender: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-02T00:56:28.76327003+09:00","level":"INFO","msg":"stream: closed","id":"lw2bm1l6"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8a7b1326ec219e3beb3d8af55a1f20d3c9966596 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-01 12:59:36,540 INFO MainThread:2521108 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-01 12:59:36,540 INFO MainThread:2521108 [wandb_setup.py:_flush():81] Configure stats pid to 2521108 +2026-05-01 12:59:36,540 INFO MainThread:2521108 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-01 12:59:36,540 INFO MainThread:2521108 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug.log +2026-05-01 12:59:36,540 INFO MainThread:2521108 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/logs/debug-internal.log +2026-05-01 12:59:36,540 INFO MainThread:2521108 [wandb_init.py:init():844] calling init triggers +2026-05-01 12:59:36,540 INFO MainThread:2521108 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '600', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-01 12:59:36,540 INFO MainThread:2521108 [wandb_init.py:init():892] starting backend +2026-05-01 12:59:37,069 INFO MainThread:2521108 [wandb_init.py:init():895] sending inform_init request +2026-05-01 12:59:37,110 INFO MainThread:2521108 [wandb_init.py:init():903] backend started and connected +2026-05-01 12:59:37,115 INFO MainThread:2521108 [wandb_init.py:init():973] updated telemetry +2026-05-01 12:59:37,156 INFO MainThread:2521108 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-01 12:59:38,174 INFO MainThread:2521108 [wandb_init.py:init():1042] starting run threads in backend +2026-05-01 12:59:38,977 INFO MainThread:2521108 [wandb_run.py:_console_start():2524] atexit reg +2026-05-01 12:59:38,977 INFO MainThread:2521108 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-01 12:59:38,977 INFO MainThread:2521108 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-01 12:59:38,977 INFO MainThread:2521108 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-01 12:59:38,985 INFO MainThread:2521108 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-02 00:56:27,100 INFO wandb-AsyncioManager-main:2521108 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-02 00:56:27,101 INFO wandb-AsyncioManager-main:2521108 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/run-lw2bm1l6.wandb b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/run-lw2bm1l6.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d6102b083129587b6140e683e85073f4b4745c09 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260501_125936-lw2bm1l6/run-lw2bm1l6.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f52c55e0231bd555df39b7e090b3ab6e6295df171589bc4d93e94b5c465d8b76 +size 9494900 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/config.yaml b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7dfac96429d91a3b9bdbca01f6915ea08fc6ab4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/config.yaml @@ -0,0 +1,390 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + wwelpjm63fo275ghix3t7j87r0vw87x0: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769_no_reward.py + - '-' + - trainer.resume=True + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_t769_no_reward + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=700 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1175596597248" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward + startedAt: "2026-05-03T23:10:11.016594Z" + writerId: wwelpjm63fo275ghix3t7j87r0vw87x0 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "False" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "0" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "700" + offload_module_in_decoding: "False" + resume: "True" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_0a9e7fd50394d8317c8b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_0a9e7fd50394d8317c8b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..283c7711d415321dd5f6657f949d5f5e0f76333b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_0a9e7fd50394d8317c8b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a9e7fd50394d8317c8bc55c367cf912a0f675c7c4403bfb00e6fb37947cbc15 +size 205917 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_195f968c26d2d073200e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_195f968c26d2d073200e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d21573ad1a4dd3d25e5795c928ef2780853e3788 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_195f968c26d2d073200e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:195f968c26d2d073200e7d141eed27288d6a8821148b8b6c246bc21e9171caf7 +size 257491 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_279df03ae1fa72af4482.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_279df03ae1fa72af4482.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ece81f373b4b5841367147424cb868cea0844bb5 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_279df03ae1fa72af4482.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:279df03ae1fa72af4482c0d8f27e9f6533243e3ffdb60761b9e3770b6d26ad78 +size 200194 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_3b5f4dc6836b99443638.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_3b5f4dc6836b99443638.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a0a3f3302d5823230561af6beff30a83fe6da3db --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_3b5f4dc6836b99443638.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b5f4dc6836b994436382440658bf597e535e26ad49f28f4c72f6575fde916b4 +size 185442 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_742edf4db049b9a7a78c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_742edf4db049b9a7a78c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..821d8bf5d3235a9e188077fdf0dfda4b79661572 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_742edf4db049b9a7a78c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:742edf4db049b9a7a78cab236fa5f79cd4b89f5ae7bf78bda169b3d55fb54687 +size 298771 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_8dad6b26232c7833b2a6.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_8dad6b26232c7833b2a6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eff75b301743c40153bfb88aa52a8c5b77f946a9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_8dad6b26232c7833b2a6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dad6b26232c7833b2a6e0eb9d260b4eb2fc2c3e71f6a58a799a0d56ea3ec749 +size 214021 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_9f9787ff0ab4e25b8d0d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_9f9787ff0ab4e25b8d0d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3110adac123641bbd8dc740ddae3fe224b2cf6d0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_9f9787ff0ab4e25b8d0d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f9787ff0ab4e25b8d0d6d060b74351396b141c82ed05730ac2cf434b0bf196f +size 245561 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_c77651e7527fa1dcdf2c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_c77651e7527fa1dcdf2c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b9be8c733f0c83525228baec87d957f6eee16bf8 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_c77651e7527fa1dcdf2c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77651e7527fa1dcdf2c2d0d4cc4b28999261b7df746eb3810eadaf0b9a0fa87 +size 237521 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_d5a22a2ad522cb8f20aa.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_d5a22a2ad522cb8f20aa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b69242d7bddd6928a55f17c8f616c16e0b0fdf4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_d5a22a2ad522cb8f20aa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5a22a2ad522cb8f20aad353e2b6ae96dc1a895b543edb553fd76468bf8586a0 +size 299145 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_de88448d1dfb6875a6d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_de88448d1dfb6875a6d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd2139fbd1efc633610ed138e64042e80603b6b6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/generated_700_de88448d1dfb6875a6d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de88448d1dfb6875a6d008c17358b947a6b5256e47e8b8a0e0ad904f48ae16e2 +size 215913 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/output.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ccc558214bcb70297c050ccef67d717924ef1c06 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/output.log @@ -0,0 +1,1828 @@ +[May 4, 2026 - 08:10:12 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 4, 2026 - 08:10:12 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 4, 2026 - 08:10:12 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 4, 2026 - 08:10:12 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 4, 2026 - 08:10:12 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 4, 2026 - 08:10:12 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 4, 2026 - 08:10:12 | INFO | fastgen.trainer:run:77 ] Starting training +[May 4, 2026 - 08:10:12 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 4, 2026 - 08:10:12 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 4, 2026 - 08:10:38 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 4, 2026 - 08:10:50 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 4, 2026 - 08:10:50 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 4, 2026 - 08:10:50 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 4, 2026 - 08:10:50 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 4, 2026 - 08:11:34 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 4, 2026 - 08:11:34 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 4, 2026 - 08:11:34 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 4, 2026 - 08:11:34 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 4, 2026 - 08:11:34 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 4, 2026 - 08:11:34 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 4, 2026 - 08:11:36 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 4, 2026 - 08:11:36 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 4, 2026 - 08:11:36 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 4, 2026 - 08:11:36 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 4, 2026 - 08:11:38 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 4, 2026 - 08:11:38 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.0s +[May 4, 2026 - 08:11:38 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 4, 2026 - 08:11:38 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 4, 2026 - 08:11:50 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 4, 2026 - 08:11:50 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.5s +[May 4, 2026 - 08:11:50 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 4, 2026 - 08:11:50 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 4, 2026 - 08:11:50 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 4, 2026 - 08:11:50 | INFO | fastgen.trainer:run:137 ] Loading checkpoints for resuming .. +[May 4, 2026 - 08:11:50 | INFO | fastgen.utils.checkpointer:load:436 ] Loading FSDP model from prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000600 +[May 4, 2026 - 08:11:50 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key net... +[May 4, 2026 - 08:11:57 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key fake_score... +[May 4, 2026 - 08:12:04 | INFO | fastgen.utils.checkpointer:load:507 ] Loading the FSDP optimizer dict for key net... +[May 4, 2026 - 08:12:04 | SUCCESS | fastgen.utils.checkpointer:load:526 ] Successfully loaded optimizer state for net +[May 4, 2026 - 08:12:04 | INFO | fastgen.utils.checkpointer:load:507 ] Loading the FSDP optimizer dict for key fake_score... +[May 4, 2026 - 08:12:04 | SUCCESS | fastgen.utils.checkpointer:load:526 ] Successfully loaded optimizer state for fake_score +[May 4, 2026 - 08:12:04 | INFO | fastgen.utils.checkpointer:load:546 ] Loading the scheduler_dict... +[May 4, 2026 - 08:12:04 | INFO | fastgen.utils.checkpointer:load:554 ] Loading the gradient scaler... +[May 4, 2026 - 08:12:04 | WARNING | fastgen.utils.checkpointer:load:559 ] Gradient scaler state is empty (likely saved from disabled scaler), skipping load. +[May 4, 2026 - 08:12:04 | INFO | fastgen.utils.checkpointer:load:562 ] Loading the callbacks... +[May 4, 2026 - 08:12:04 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 600. +[May 4, 2026 - 08:12:04 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17136.85 M total params (logical). +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4285.56 M total params LOCAL on rank 0. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 1. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 2. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 3. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 4, 2026 - 08:12:10 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 4, 2026 - 08:12:10 | INFO | fastgen.trainer:run:174 ] iter_start: 600 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.371189 37.407166 37.284412 +peak_gpu_mem_gb 58.334157 58.335719 58.333636 +peak_gpu_mem_reserved_gb 60.547363 60.548828 60.546875 +util 82.000000 87.000000 78.000000 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 601 : data loading time 1.21 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 601 : avg forward pass time 10.13 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 601 : backward pass time 10.12 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 601 : optimizer step time 1.75 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 601-------------------- +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:12:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.545963 37.581825 37.459202 +peak_gpu_mem_gb 60.982488 60.984831 60.981707 +peak_gpu_mem_reserved_gb 62.289551 62.291016 62.289062 +util 92.750000 100.000000 88.000000 +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 602 : avg iteration time 43.29 seconds +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 602 : data loading time 0.00 +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 602 : avg forward pass time 10.69 +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 602 : backward pass time 10.17 +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 602 : optimizer step time 1.58 +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 602-------------------- +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:13:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.546363 37.581963 37.459991 +peak_gpu_mem_gb 60.982488 60.984831 60.981707 +peak_gpu_mem_reserved_gb 62.328613 62.330078 62.328125 +util 95.500000 99.000000 92.000000 +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 603 : avg iteration time 48.48 seconds +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 603 : data loading time 0.00 +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 603 : avg forward pass time 13.29 +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 603 : backward pass time 10.17 +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 603 : optimizer step time 1.58 +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 603-------------------- +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:14:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.546763 37.582146 37.461174 +peak_gpu_mem_gb 60.982488 60.984831 60.981707 +peak_gpu_mem_reserved_gb 62.348145 62.349609 62.347656 +util 96.500000 100.000000 90.000000 +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 604 : avg iteration time 43.27 seconds +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 604 : data loading time 0.00 +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 604 : avg forward pass time 10.69 +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 604 : backward pass time 10.14 +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 604 : optimizer step time 1.57 +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 604-------------------- +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:15:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.562650 37.597958 37.477062 +peak_gpu_mem_gb 96.120852 96.123195 96.120071 +peak_gpu_mem_reserved_gb 103.259766 103.283203 103.189453 +util 57.500000 61.000000 50.000000 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 605 : avg iteration time 163.62 seconds +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 605 : data loading time 0.00 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 605 : avg forward pass time 72.89 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 605 : backward pass time 8.50 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 605 : optimizer step time 0.44 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 605-------------------- +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5093 iter count: 1.0 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5093 iter count: 1.0 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5093 iter count: 1.0 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5093iter count: 1.0 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:17:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.562807 37.598362 37.477074 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.934570 64.996094 64.914062 +util 96.750000 99.000000 93.000000 +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 606 : avg iteration time 46.93 seconds +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 606 : data loading time 0.00 +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 606 : avg forward pass time 12.53 +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 606 : backward pass time 10.15 +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 606 : optimizer step time 1.58 +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 606-------------------- +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:18:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.563274 37.599537 37.477081 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.851562 64.898438 64.835938 +util 94.500000 99.000000 90.000000 +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 607 : avg iteration time 43.37 seconds +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 607 : data loading time 0.00 +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 607 : avg forward pass time 10.75 +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 607 : backward pass time 10.16 +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 607 : optimizer step time 1.56 +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 607-------------------- +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:19:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.563343 37.599598 37.477093 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.885742 64.917969 64.875000 +util 94.750000 99.000000 88.000000 +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 608 : avg iteration time 48.52 seconds +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 608 : data loading time 0.00 +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 608 : avg forward pass time 13.31 +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 608 : backward pass time 10.17 +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 608 : optimizer step time 1.56 +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 608-------------------- +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:20:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.563581 37.599838 37.477402 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.822266 65.015625 64.757812 +util 92.000000 97.000000 88.000000 +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 609 : avg iteration time 45.85 seconds +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 609 : data loading time 0.00 +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 609 : avg forward pass time 12.00 +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 609 : backward pass time 10.14 +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 609 : optimizer step time 1.58 +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 609-------------------- +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:20:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.564869 37.600784 37.478886 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.205566 105.218750 105.166016 +util 67.500000 72.000000 60.000000 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 610 : avg iteration time 160.34 seconds +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 610 : data loading time 0.00 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 610 : avg forward pass time 71.58 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 610 : backward pass time 8.40 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 610 : optimizer step time 0.40 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 610-------------------- +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5840 iter count: 1.0 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5840 iter count: 1.0 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5840 iter count: 1.0 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5840iter count: 1.0 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:23:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.564942 37.600971 37.479038 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.910156 64.957031 64.894531 +util 97.000000 100.000000 93.000000 +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 611 : avg iteration time 47.14 seconds +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 611 : data loading time 0.00 +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 611 : avg forward pass time 12.64 +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 611 : backward pass time 10.13 +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 611 : optimizer step time 1.58 +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 611-------------------- +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:24:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.565041 37.601078 37.479313 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.919922 64.937500 64.914062 +util 96.500000 99.000000 90.000000 +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 612 : avg iteration time 45.86 seconds +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 612 : data loading time 0.00 +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 612 : avg forward pass time 11.99 +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 612 : backward pass time 10.16 +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 612 : optimizer step time 1.57 +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 612-------------------- +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:25:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.565158 37.601147 37.479412 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.934570 64.996094 64.914062 +util 97.250000 100.000000 91.000000 +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 613 : avg iteration time 43.21 seconds +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 613 : data loading time 0.00 +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 613 : avg forward pass time 10.67 +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 613 : backward pass time 10.15 +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 613 : optimizer step time 1.57 +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 613-------------------- +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:25:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.565486 37.601364 37.479622 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.929688 64.976562 64.914062 +util 96.500000 100.000000 93.000000 +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 614 : avg iteration time 43.32 seconds +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 614 : data loading time 0.00 +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 614 : avg forward pass time 10.73 +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 614 : backward pass time 10.15 +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 614 : optimizer step time 1.56 +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 614-------------------- +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:26:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.565657 37.601734 37.480198 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.210449 105.218750 105.185547 +util 68.000000 72.000000 60.000000 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 615 : avg iteration time 157.28 seconds +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 615 : data loading time 0.00 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 615 : avg forward pass time 70.06 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 615 : backward pass time 8.40 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 615 : optimizer step time 0.40 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 615-------------------- +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5596 iter count: 1.0 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5596 iter count: 1.0 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5596 iter count: 1.0 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5596iter count: 1.0 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:29:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.565781 37.601948 37.480255 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.915039 65.035156 64.875000 +util 91.500000 96.000000 87.000000 +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 616 : avg iteration time 46.86 seconds +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 616 : data loading time 0.00 +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 616 : avg forward pass time 12.50 +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 616 : backward pass time 10.15 +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 616 : optimizer step time 1.58 +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 616-------------------- +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:30:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.565953 37.602215 37.480309 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.768555 64.976562 64.699219 +util 90.750000 96.000000 86.000000 +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 617 : avg iteration time 45.78 seconds +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 617 : data loading time 0.00 +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 617 : avg forward pass time 11.96 +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 617 : backward pass time 10.14 +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 617 : optimizer step time 1.57 +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 617-------------------- +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:30:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.566373 37.602230 37.480446 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.763672 65.015625 64.679688 +util 95.000000 99.000000 91.000000 +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 618 : avg iteration time 48.46 seconds +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 618 : data loading time 0.00 +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 618 : avg forward pass time 13.29 +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 618 : backward pass time 10.15 +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 618 : optimizer step time 1.59 +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 618-------------------- +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:31:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.566422 37.602234 37.480446 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.709961 64.976562 64.621094 +util 95.000000 100.000000 91.000000 +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 619 : avg iteration time 45.80 seconds +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 619 : data loading time 0.00 +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 619 : avg forward pass time 11.97 +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 619 : backward pass time 10.14 +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 619 : optimizer step time 1.57 +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 619-------------------- +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:32:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.566633 37.602432 37.480930 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.322754 105.693359 105.199219 +util 68.250000 73.000000 64.000000 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 620 : avg iteration time 162.66 seconds +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 620 : data loading time 0.00 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 620 : avg forward pass time 72.75 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 620 : backward pass time 8.38 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 620 : optimizer step time 0.40 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 620-------------------- +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5674 iter count: 1.0 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5674 iter count: 1.0 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5674 iter count: 1.0 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5674iter count: 1.0 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:35:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.566773 37.602592 37.480946 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.925293 64.958984 64.914062 +util 91.250000 96.000000 87.000000 +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 621 : avg iteration time 47.10 seconds +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 621 : data loading time 0.00 +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 621 : avg forward pass time 12.61 +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 621 : backward pass time 10.15 +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 621 : optimizer step time 1.58 +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 621-------------------- +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.566955 37.602715 37.480942 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.739746 64.978516 64.660156 +util 97.250000 100.000000 91.000000 +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 622 : avg iteration time 48.54 seconds +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 622 : data loading time 0.00 +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 622 : avg forward pass time 13.32 +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 622 : backward pass time 10.16 +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 622 : optimizer step time 1.57 +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 622-------------------- +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:36:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567019 37.602715 37.480984 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.764160 64.958984 64.699219 +util 95.500000 99.000000 92.000000 +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 623 : avg iteration time 45.91 seconds +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 623 : data loading time 0.00 +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 623 : avg forward pass time 12.01 +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 623 : backward pass time 10.14 +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 623 : optimizer step time 1.58 +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 623-------------------- +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:37:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567140 37.602837 37.481007 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.739746 64.978516 64.660156 +util 94.500000 99.000000 88.000000 +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 624 : avg iteration time 45.90 seconds +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 624 : data loading time 0.00 +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 624 : avg forward pass time 12.02 +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 624 : backward pass time 10.13 +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 624 : optimizer step time 1.58 +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 624-------------------- +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:38:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567305 37.602924 37.481060 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.381348 105.810547 105.238281 +util 65.000000 70.000000 56.000000 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 625 : avg iteration time 157.56 seconds +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 625 : data loading time 0.00 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 625 : avg forward pass time 70.18 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 625 : backward pass time 8.40 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 625 : optimizer step time 0.40 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 625-------------------- +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5293 iter count: 1.0 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5293 iter count: 1.0 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5293 iter count: 1.0 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5293iter count: 1.0 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:40:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567418 37.603058 37.481209 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.617676 65.017578 64.484375 +util 94.250000 99.000000 87.000000 +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 626 : avg iteration time 47.03 seconds +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 626 : data loading time 0.00 +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 626 : avg forward pass time 12.59 +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 626 : backward pass time 10.14 +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 626 : optimizer step time 1.59 +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 626-------------------- +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:41:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567490 37.603271 37.481270 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.588379 64.900391 64.484375 +util 93.000000 98.000000 89.000000 +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 627 : avg iteration time 45.87 seconds +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 627 : data loading time 0.00 +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 627 : avg forward pass time 11.99 +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 627 : backward pass time 10.14 +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 627 : optimizer step time 1.59 +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 627-------------------- +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:42:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567548 37.603371 37.481285 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.607910 64.919922 64.503906 +util 97.250000 100.000000 94.000000 +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 628 : avg iteration time 43.26 seconds +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 628 : data loading time 0.00 +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 628 : avg forward pass time 10.68 +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 628 : backward pass time 10.16 +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 628 : optimizer step time 1.59 +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 628-------------------- +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:43:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567596 37.603394 37.481293 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.612793 64.939453 64.503906 +util 97.750000 99.000000 95.000000 +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 629 : avg iteration time 46.01 seconds +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 629 : data loading time 0.00 +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 629 : avg forward pass time 12.06 +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 629 : backward pass time 10.14 +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 629 : optimizer step time 1.58 +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 629-------------------- +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:43:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567355 37.603569 37.481304 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.166504 105.244141 105.140625 +util 62.500000 67.000000 58.000000 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 630 : avg iteration time 157.55 seconds +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 630 : data loading time 0.00 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 630 : avg forward pass time 70.19 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 630 : backward pass time 8.40 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 630 : optimizer step time 0.40 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 630-------------------- +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5059 iter count: 1.0 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5059 iter count: 1.0 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5059 iter count: 1.0 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5059iter count: 1.0 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:46:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567479 37.603642 37.481373 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.919922 64.937500 64.914062 +util 97.250000 100.000000 91.000000 +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 631 : avg iteration time 44.30 seconds +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 631 : data loading time 0.00 +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 631 : avg forward pass time 11.22 +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 631 : backward pass time 10.14 +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 631 : optimizer step time 1.56 +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 631-------------------- +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0151 iter count: 1.0 +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0151 iter count: 1.0 +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:47:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567634 37.603683 37.481373 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.724609 64.917969 64.660156 +util 94.750000 99.000000 91.000000 +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 632 : avg iteration time 43.30 seconds +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 632 : data loading time 0.00 +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 632 : avg forward pass time 10.71 +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 632 : backward pass time 10.15 +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 632 : optimizer step time 1.58 +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 632-------------------- +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:47:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567704 37.603821 37.481380 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749023 64.898438 64.699219 +util 95.750000 99.000000 88.000000 +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 633 : avg iteration time 43.27 seconds +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 633 : data loading time 0.00 +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 633 : avg forward pass time 10.69 +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 633 : backward pass time 10.16 +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 633 : optimizer step time 1.59 +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 633-------------------- +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:48:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567695 37.603821 37.481380 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.773438 64.937500 64.718750 +util 94.500000 99.000000 88.000000 +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 634 : avg iteration time 46.05 seconds +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 634 : data loading time 0.00 +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 634 : avg forward pass time 12.09 +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 634 : backward pass time 10.13 +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 634 : optimizer step time 1.58 +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 634-------------------- +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567369 37.603851 37.481388 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.215332 105.238281 105.146484 +util 67.500000 72.000000 59.000000 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 635 : avg iteration time 159.99 seconds +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 635 : data loading time 0.00 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 635 : avg forward pass time 71.39 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 635 : backward pass time 8.41 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 635 : optimizer step time 0.40 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 635-------------------- +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5605 iter count: 1.0 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5605 iter count: 1.0 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5605 iter count: 1.0 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5605iter count: 1.0 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:52:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567440 37.603851 37.481430 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.934570 64.937500 64.933594 +util 92.750000 100.000000 88.000000 +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 636 : avg iteration time 49.64 seconds +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 636 : data loading time 0.00 +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 636 : avg forward pass time 13.89 +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 636 : backward pass time 10.15 +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 636 : optimizer step time 1.57 +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 636-------------------- +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:52:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567825 37.603828 37.481537 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 65.178711 65.972656 64.914062 +util 98.250000 100.000000 95.000000 +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 637 : avg iteration time 45.91 seconds +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 637 : data loading time 0.00 +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 637 : avg forward pass time 12.03 +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 637 : backward pass time 10.13 +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 637 : optimizer step time 1.56 +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 637-------------------- +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:53:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568035 37.603870 37.481579 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.802734 64.878906 64.777344 +util 98.000000 100.000000 95.000000 +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 638 : avg iteration time 46.00 seconds +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 638 : data loading time 0.00 +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 638 : avg forward pass time 12.08 +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 638 : backward pass time 10.14 +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 638 : optimizer step time 1.56 +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 638-------------------- +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:54:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568065 37.603874 37.481689 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.937500 64.777344 +util 95.750000 99.000000 88.000000 +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 639 : avg iteration time 45.82 seconds +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 639 : data loading time 0.00 +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 639 : avg forward pass time 11.97 +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 639 : backward pass time 10.15 +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 639 : optimizer step time 1.59 +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 639-------------------- +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:55:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568227 37.603909 37.481808 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.229980 105.238281 105.205078 +util 59.500000 64.000000 56.000000 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 640 : avg iteration time 160.08 seconds +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 640 : data loading time 0.00 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 640 : avg forward pass time 71.46 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 640 : backward pass time 8.40 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 640 : optimizer step time 0.40 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 640-------------------- +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5625 iter count: 1.0 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5625 iter count: 1.0 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5625 iter count: 1.0 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5625iter count: 1.0 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:57:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568003 37.603966 37.481831 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.612305 64.937500 64.503906 +util 92.750000 97.000000 89.000000 +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 641 : avg iteration time 46.82 seconds +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 641 : data loading time 0.00 +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 641 : avg forward pass time 12.49 +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 641 : backward pass time 10.13 +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 641 : optimizer step time 1.57 +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 641-------------------- +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:58:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567995 37.603970 37.481808 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.832031 64.937500 64.796875 +util 95.500000 100.000000 87.000000 +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 642 : avg iteration time 45.89 seconds +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 642 : data loading time 0.00 +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 642 : avg forward pass time 12.00 +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 642 : backward pass time 10.15 +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 642 : optimizer step time 1.58 +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 642-------------------- +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 08:59:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568014 37.604031 37.481808 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.822266 64.957031 64.777344 +util 92.500000 100.000000 88.000000 +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 643 : avg iteration time 45.86 seconds +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 643 : data loading time 0.00 +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 643 : avg forward pass time 11.99 +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 643 : backward pass time 10.15 +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 643 : optimizer step time 1.58 +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 643-------------------- +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:00:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568007 37.603992 37.481808 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.827148 64.976562 64.777344 +util 91.750000 96.000000 88.000000 +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 644 : avg iteration time 45.84 seconds +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 644 : data loading time 0.00 +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 644 : avg forward pass time 11.99 +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 644 : backward pass time 10.14 +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 644 : optimizer step time 1.57 +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 644-------------------- +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:00:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568018 37.604012 37.481823 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.381348 105.751953 105.257812 +util 67.750000 71.000000 65.000000 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 645 : avg iteration time 159.92 seconds +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 645 : data loading time 0.00 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 645 : avg forward pass time 71.39 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 645 : backward pass time 8.39 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 645 : optimizer step time 0.40 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 645-------------------- +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5342 iter count: 1.0 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5342 iter count: 1.0 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5342 iter count: 1.0 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5342iter count: 1.0 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:03:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.567994 37.604061 37.481861 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.934570 64.996094 64.914062 +util 91.750000 96.000000 88.000000 +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 646 : avg iteration time 44.29 seconds +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 646 : data loading time 0.00 +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 646 : avg forward pass time 11.21 +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 646 : backward pass time 10.14 +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 646 : optimizer step time 1.59 +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 646-------------------- +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:04:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568001 37.604076 37.481873 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.695312 64.917969 64.621094 +util 95.500000 99.000000 92.000000 +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 647 : avg iteration time 45.84 seconds +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 647 : data loading time 0.00 +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 647 : avg forward pass time 11.99 +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 647 : backward pass time 10.15 +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 647 : optimizer step time 1.58 +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 647-------------------- +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:05:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568010 37.604076 37.481888 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.729492 64.937500 64.660156 +util 98.000000 100.000000 95.000000 +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 648 : avg iteration time 45.84 seconds +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 648 : data loading time 0.00 +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 648 : avg forward pass time 11.99 +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 648 : backward pass time 10.14 +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 648 : optimizer step time 1.57 +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 648-------------------- +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:05:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568300 37.604084 37.481846 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.729492 64.937500 64.660156 +util 98.750000 100.000000 96.000000 +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 649 : avg iteration time 48.41 seconds +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 649 : data loading time 0.00 +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 649 : avg forward pass time 13.27 +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 649 : backward pass time 10.14 +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 649 : optimizer step time 1.58 +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 649-------------------- +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:06:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568138 37.604153 37.481956 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.273926 105.277344 105.263672 +util 62.500000 71.000000 57.000000 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 650 : avg iteration time 162.88 seconds +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 650 : data loading time 0.00 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 650 : avg forward pass time 72.86 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 650 : backward pass time 8.40 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 650 : optimizer step time 0.40 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 650-------------------- +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4849 iter count: 1.0 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4849 iter count: 1.0 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4849 iter count: 1.0 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4849iter count: 1.0 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:09:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568131 37.604153 37.481911 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.998047 65.031250 64.898438 +util 97.750000 100.000000 91.000000 +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 651 : avg iteration time 49.57 seconds +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 651 : data loading time 0.00 +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 651 : avg forward pass time 13.85 +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 651 : backward pass time 10.15 +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 651 : optimizer step time 1.58 +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 651-------------------- +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:10:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568177 37.604164 37.482059 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.919922 64.933594 64.878906 +util 95.500000 99.000000 87.000000 +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 652 : avg iteration time 45.95 seconds +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 652 : data loading time 0.00 +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 652 : avg forward pass time 12.05 +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 652 : backward pass time 10.15 +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 652 : optimizer step time 1.56 +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 652-------------------- +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:11:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568174 37.604172 37.482059 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.602539 64.898438 64.503906 +util 96.250000 100.000000 88.000000 +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 653 : avg iteration time 45.81 seconds +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 653 : data loading time 0.00 +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 653 : avg forward pass time 11.98 +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 653 : backward pass time 10.14 +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 653 : optimizer step time 1.57 +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 653-------------------- +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:11:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568369 37.604176 37.482155 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.993164 65.011719 64.937500 +util 92.250000 99.000000 88.000000 +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 654 : avg iteration time 45.85 seconds +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 654 : data loading time 0.00 +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 654 : avg forward pass time 12.00 +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 654 : backward pass time 10.13 +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 654 : optimizer step time 1.58 +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 654-------------------- +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:12:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568195 37.604176 37.482262 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.239746 105.244141 105.238281 +util 67.250000 72.000000 59.000000 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 655 : avg iteration time 159.97 seconds +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 655 : data loading time 0.00 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 655 : avg forward pass time 71.40 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 655 : backward pass time 8.39 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 655 : optimizer step time 0.40 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 655-------------------- +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5469 iter count: 1.0 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5469 iter count: 1.0 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5469 iter count: 1.0 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5469iter count: 1.0 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:15:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568207 37.604153 37.482285 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.978516 65.011719 64.878906 +util 95.500000 99.000000 92.000000 +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 656 : avg iteration time 49.51 seconds +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 656 : data loading time 0.00 +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 656 : avg forward pass time 13.82 +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 656 : backward pass time 10.15 +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 656 : optimizer step time 1.57 +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 656-------------------- +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:16:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568277 37.604153 37.482285 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.973633 65.031250 64.800781 +util 96.750000 100.000000 89.000000 +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 657 : avg iteration time 45.88 seconds +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 657 : data loading time 0.00 +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 657 : avg forward pass time 12.00 +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 657 : backward pass time 10.14 +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 657 : optimizer step time 1.56 +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 657-------------------- +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:16:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568271 37.604153 37.482285 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.880859 64.957031 64.855469 +util 97.250000 100.000000 90.000000 +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 658 : avg iteration time 45.81 seconds +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 658 : data loading time 0.00 +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 658 : avg forward pass time 11.97 +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 658 : backward pass time 10.14 +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 658 : optimizer step time 1.57 +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 658-------------------- +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:17:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568429 37.604179 37.482292 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.963867 64.992188 64.878906 +util 90.750000 95.000000 87.000000 +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 659 : avg iteration time 48.46 seconds +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 659 : data loading time 0.00 +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 659 : avg forward pass time 13.29 +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 659 : backward pass time 10.15 +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 659 : optimizer step time 1.57 +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 659-------------------- +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:18:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568213 37.604244 37.482399 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.244629 105.277344 105.146484 +util 61.250000 70.000000 56.000000 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 660 : avg iteration time 157.38 seconds +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 660 : data loading time 0.00 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 660 : avg forward pass time 70.11 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 660 : backward pass time 8.40 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 660 : optimizer step time 0.40 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 660-------------------- +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4912 iter count: 1.0 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4912 iter count: 1.0 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4912 iter count: 1.0 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4912iter count: 1.0 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:21:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568217 37.604256 37.482403 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.905273 64.937500 64.894531 +util 94.750000 99.000000 91.000000 +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 661 : avg iteration time 46.87 seconds +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 661 : data loading time 0.00 +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 661 : avg forward pass time 12.50 +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 661 : backward pass time 10.15 +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 661 : optimizer step time 1.58 +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 661-------------------- +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:21:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568361 37.604256 37.482517 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.890625 64.937500 64.875000 +util 96.250000 99.000000 93.000000 +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 662 : avg iteration time 45.82 seconds +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 662 : data loading time 0.00 +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 662 : avg forward pass time 11.98 +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 662 : backward pass time 10.13 +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 662 : optimizer step time 1.58 +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 662-------------------- +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:22:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568365 37.604256 37.482555 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.875977 64.878906 64.875000 +util 96.750000 100.000000 89.000000 +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 663 : avg iteration time 48.47 seconds +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 663 : data loading time 0.00 +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 663 : avg forward pass time 13.31 +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 663 : backward pass time 10.13 +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 663 : optimizer step time 1.58 +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 663-------------------- +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:23:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568356 37.604263 37.482510 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.875977 64.878906 64.875000 +util 91.750000 100.000000 87.000000 +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 664 : avg iteration time 48.44 seconds +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 664 : data loading time 0.00 +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 664 : avg forward pass time 13.28 +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 664 : backward pass time 10.16 +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 664 : optimizer step time 1.58 +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 664-------------------- +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:24:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568311 37.604279 37.482529 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.244629 105.263672 105.238281 +util 64.500000 68.000000 61.000000 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 665 : avg iteration time 154.69 seconds +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 665 : data loading time 0.00 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 665 : avg forward pass time 68.76 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 665 : backward pass time 8.40 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 665 : optimizer step time 0.39 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 665-------------------- +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5107 iter count: 1.0 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5107 iter count: 1.0 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5107 iter count: 1.0 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5107iter count: 1.0 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:26:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568323 37.604294 37.482571 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.949219 65.054688 64.914062 +util 96.000000 100.000000 88.000000 +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 666 : avg iteration time 46.86 seconds +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 666 : data loading time 0.00 +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 666 : avg forward pass time 12.49 +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 666 : backward pass time 10.15 +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 666 : optimizer step time 1.58 +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 666-------------------- +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:27:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568545 37.604290 37.482582 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.880859 64.898438 64.875000 +util 92.500000 100.000000 88.000000 +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 667 : avg iteration time 45.85 seconds +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 667 : data loading time 0.00 +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 667 : avg forward pass time 11.98 +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 667 : backward pass time 10.16 +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 667 : optimizer step time 1.57 +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 667-------------------- +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:28:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568578 37.604294 37.482613 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 64.917969 64.894531 +util 94.500000 98.000000 91.000000 +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 668 : avg iteration time 43.27 seconds +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 668 : data loading time 0.00 +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 668 : avg forward pass time 10.70 +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 668 : backward pass time 10.15 +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 668 : optimizer step time 1.56 +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 668-------------------- +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:29:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568707 37.604298 37.482635 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 64.917969 64.894531 +util 98.750000 100.000000 97.000000 +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 669 : avg iteration time 48.47 seconds +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 669 : data loading time 0.00 +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 669 : avg forward pass time 13.30 +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 669 : backward pass time 10.15 +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 669 : optimizer step time 1.57 +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 669-------------------- +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:29:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568186 37.604294 37.482616 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.278809 105.341797 105.257812 +util 62.500000 66.000000 59.000000 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 670 : avg iteration time 162.93 seconds +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 670 : data loading time 0.00 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 670 : avg forward pass time 72.89 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 670 : backward pass time 8.39 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 670 : optimizer step time 0.40 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 670-------------------- +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5161 iter count: 1.0 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5161 iter count: 1.0 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5161 iter count: 1.0 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5161iter count: 1.0 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568194 37.604309 37.482616 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.846680 64.937500 64.816406 +util 92.250000 99.000000 88.000000 +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 671 : avg iteration time 44.33 seconds +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 671 : data loading time 0.00 +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 671 : avg forward pass time 11.23 +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 671 : backward pass time 10.15 +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 671 : optimizer step time 1.58 +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 671-------------------- +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0143 iter count: 1.0 +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:33:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568332 37.604324 37.482662 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.846680 64.937500 64.816406 +util 95.250000 99.000000 92.000000 +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 672 : avg iteration time 45.88 seconds +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 672 : data loading time 0.00 +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 672 : avg forward pass time 12.00 +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 672 : backward pass time 10.15 +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 672 : optimizer step time 1.58 +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 672-------------------- +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:34:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568452 37.604317 37.482700 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.846680 64.937500 64.816406 +util 96.750000 100.000000 89.000000 +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 673 : avg iteration time 48.48 seconds +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 673 : data loading time 0.00 +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 673 : avg forward pass time 13.30 +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 673 : backward pass time 10.16 +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 673 : optimizer step time 1.58 +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 673-------------------- +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:34:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568457 37.604340 37.482677 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.563477 64.917969 64.445312 +util 91.500000 99.000000 87.000000 +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 674 : avg iteration time 43.23 seconds +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 674 : data loading time 0.00 +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 674 : avg forward pass time 10.68 +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 674 : backward pass time 10.16 +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 674 : optimizer step time 1.57 +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 674-------------------- +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:35:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568397 37.604366 37.482689 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.181152 105.185547 105.179688 +util 66.750000 71.000000 63.000000 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 675 : avg iteration time 160.12 seconds +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 675 : data loading time 0.00 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 675 : avg forward pass time 71.48 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 675 : backward pass time 8.39 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 675 : optimizer step time 0.40 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 675-------------------- +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5547 iter count: 1.0 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5547 iter count: 1.0 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5547 iter count: 1.0 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5547iter count: 1.0 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:38:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568399 37.604347 37.482689 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.832031 65.054688 64.757812 +util 91.750000 96.000000 88.000000 +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 676 : avg iteration time 49.52 seconds +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 676 : data loading time 0.00 +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 676 : avg forward pass time 13.83 +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 676 : backward pass time 10.16 +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 676 : optimizer step time 1.56 +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 676-------------------- +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:39:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568591 37.604347 37.482693 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.812500 65.035156 64.738281 +util 98.750000 100.000000 97.000000 +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 677 : avg iteration time 48.50 seconds +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 677 : data loading time 0.00 +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 677 : avg forward pass time 13.31 +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 677 : backward pass time 10.15 +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 677 : optimizer step time 1.57 +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 677-------------------- +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:39:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568784 37.604355 37.482693 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.763672 65.074219 64.660156 +util 90.250000 94.000000 87.000000 +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 678 : avg iteration time 45.88 seconds +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 678 : data loading time 0.00 +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 678 : avg forward pass time 12.02 +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 678 : backward pass time 10.14 +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 678 : optimizer step time 1.58 +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 678-------------------- +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:40:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568795 37.604343 37.482750 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.937500 64.777344 +util 90.500000 94.000000 87.000000 +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 679 : avg iteration time 43.20 seconds +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 679 : data loading time 0.00 +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 679 : avg forward pass time 10.67 +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 679 : backward pass time 10.14 +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 679 : optimizer step time 1.57 +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 679-------------------- +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:41:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568274 37.604370 37.482758 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.278809 105.296875 105.224609 +util 62.250000 66.000000 59.000000 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 680 : avg iteration time 157.24 seconds +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 680 : data loading time 0.00 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 680 : avg forward pass time 70.03 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 680 : backward pass time 8.39 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 680 : optimizer step time 0.40 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 680-------------------- +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4902 iter count: 1.0 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4902 iter count: 1.0 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4902 iter count: 1.0 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4902iter count: 1.0 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:43:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568302 37.604347 37.482868 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 65.022461 65.054688 65.011719 +util 97.750000 100.000000 95.000000 +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 681 : avg iteration time 44.26 seconds +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 681 : data loading time 0.00 +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 681 : avg forward pass time 11.20 +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 681 : backward pass time 10.13 +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 681 : optimizer step time 1.58 +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 681-------------------- +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:44:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568419 37.604359 37.482849 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.592773 64.976562 64.464844 +util 96.500000 100.000000 88.000000 +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 682 : avg iteration time 45.84 seconds +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 682 : data loading time 0.00 +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 682 : avg forward pass time 11.99 +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 682 : backward pass time 10.14 +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 682 : optimizer step time 1.58 +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 682-------------------- +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:45:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568740 37.604385 37.482849 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.885742 65.035156 64.835938 +util 91.500000 99.000000 87.000000 +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 683 : avg iteration time 45.84 seconds +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 683 : data loading time 0.00 +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 683 : avg forward pass time 11.99 +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 683 : backward pass time 10.14 +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 683 : optimizer step time 1.57 +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 683-------------------- +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:46:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568729 37.604366 37.482849 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 65.012695 65.074219 64.992188 +util 92.000000 99.000000 88.000000 +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 684 : avg iteration time 45.80 seconds +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 684 : data loading time 0.00 +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 684 : avg forward pass time 11.97 +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 684 : backward pass time 10.12 +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 684 : optimizer step time 1.58 +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 684-------------------- +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:47:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568748 37.604370 37.482872 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.220215 105.224609 105.218750 +util 67.750000 72.000000 59.000000 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 685 : avg iteration time 162.58 seconds +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 685 : data loading time 0.00 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 685 : avg forward pass time 72.71 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 685 : backward pass time 8.39 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 685 : optimizer step time 0.40 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 685-------------------- +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5371 iter count: 1.0 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5371 iter count: 1.0 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5371 iter count: 1.0 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5371iter count: 1.0 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:49:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568546 37.604370 37.482853 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.924805 64.957031 64.914062 +util 92.000000 100.000000 88.000000 +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 686 : avg iteration time 44.22 seconds +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 686 : data loading time 0.00 +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 686 : avg forward pass time 11.18 +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 686 : backward pass time 10.15 +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 686 : optimizer step time 1.57 +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 686-------------------- +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:50:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568565 37.604374 37.482903 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.573242 65.015625 64.425781 +util 97.500000 100.000000 91.000000 +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 687 : avg iteration time 43.19 seconds +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 687 : data loading time 0.00 +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 687 : avg forward pass time 10.67 +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 687 : backward pass time 10.13 +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 687 : optimizer step time 1.56 +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 687-------------------- +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:51:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568622 37.604374 37.482895 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.543945 64.898438 64.425781 +util 95.750000 99.000000 87.000000 +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 688 : avg iteration time 43.16 seconds +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 688 : data loading time 0.00 +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 688 : avg forward pass time 10.65 +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 688 : backward pass time 10.15 +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 688 : optimizer step time 1.56 +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 688-------------------- +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:51:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568987 37.604374 37.482883 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.563477 64.917969 64.445312 +util 91.250000 95.000000 88.000000 +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 689 : avg iteration time 48.49 seconds +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 689 : data loading time 0.00 +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 689 : avg forward pass time 13.31 +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 689 : backward pass time 10.14 +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 689 : optimizer step time 1.57 +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 689-------------------- +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:52:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568735 37.604374 37.482906 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.166504 105.185547 105.160156 +util 66.000000 69.000000 63.000000 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 690 : avg iteration time 160.47 seconds +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 690 : data loading time 0.00 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 690 : avg forward pass time 71.65 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 690 : backward pass time 8.39 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 690 : optimizer step time 0.40 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 690-------------------- +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5420 iter count: 1.0 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5420 iter count: 1.0 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5420 iter count: 1.0 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5420iter count: 1.0 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:55:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568759 37.604374 37.483009 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.890625 64.937500 64.875000 +util 90.250000 94.000000 87.000000 +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 691 : avg iteration time 44.32 seconds +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 691 : data loading time 0.00 +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 691 : avg forward pass time 11.22 +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 691 : backward pass time 10.16 +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 691 : optimizer step time 1.57 +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 691-------------------- +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:56:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568776 37.604393 37.483036 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.758789 64.937500 64.699219 +util 91.000000 94.000000 88.000000 +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 692 : avg iteration time 45.81 seconds +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 692 : data loading time 0.00 +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 692 : avg forward pass time 11.97 +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 692 : backward pass time 10.12 +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 692 : optimizer step time 1.58 +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 692-------------------- +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:56:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568850 37.604485 37.483047 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.797852 65.093750 64.699219 +util 95.500000 99.000000 92.000000 +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 693 : avg iteration time 43.25 seconds +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 693 : data loading time 0.00 +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 693 : avg forward pass time 10.68 +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 693 : backward pass time 10.16 +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 693 : optimizer step time 1.57 +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 693-------------------- +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:57:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.569088 37.604485 37.483040 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734375 64.957031 64.660156 +util 98.000000 100.000000 95.000000 +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 694 : avg iteration time 48.44 seconds +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 694 : data loading time 0.00 +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 694 : avg forward pass time 13.28 +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 694 : backward pass time 10.15 +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 694 : optimizer step time 1.58 +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 694-------------------- +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 09:58:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568864 37.604485 37.483059 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.332520 105.355469 105.263672 +util 66.750000 70.000000 63.000000 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 695 : avg iteration time 160.21 seconds +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 695 : data loading time 0.00 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 695 : avg forward pass time 71.53 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 695 : backward pass time 8.40 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 695 : optimizer step time 0.40 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 695-------------------- +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4536 iter count: 1.0 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4536 iter count: 1.0 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4536 iter count: 1.0 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4536iter count: 1.0 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 10:01:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.568873 37.604496 37.483063 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.988281 64.992188 64.976562 +util 97.250000 100.000000 91.000000 +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 696 : avg iteration time 44.26 seconds +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 696 : data loading time 0.00 +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 696 : avg forward pass time 11.20 +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 696 : backward pass time 10.14 +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 696 : optimizer step time 1.58 +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 696-------------------- +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 10:01:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.569068 37.604515 37.483086 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.963867 64.972656 64.937500 +util 92.250000 96.000000 89.000000 +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 697 : avg iteration time 45.86 seconds +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 697 : data loading time 0.00 +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 697 : avg forward pass time 12.00 +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 697 : backward pass time 10.14 +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 697 : optimizer step time 1.57 +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 697-------------------- +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 10:02:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.569066 37.604530 37.483063 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.895508 64.898438 64.894531 +util 92.000000 99.000000 88.000000 +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 698 : avg iteration time 43.35 seconds +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 698 : data loading time 0.00 +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 698 : avg forward pass time 10.76 +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 698 : backward pass time 10.15 +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 698 : optimizer step time 1.56 +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 698-------------------- +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 10:03:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.569140 37.604530 37.483109 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.895508 64.898438 64.894531 +util 97.000000 100.000000 89.000000 +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 699 : avg iteration time 45.76 seconds +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 699 : data loading time 0.00 +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 699 : avg forward pass time 11.96 +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 699 : backward pass time 10.13 +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 699 : optimizer step time 1.56 +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 699-------------------- +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 10:04:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 10:04:04 | INFO | fastgen.trainer:run:225 ] Training complete. +[May 4, 2026 - 10:04:04 | INFO | fastgen.trainer:validate:358 ] Validation iteration 700 +[May 4, 2026 - 10:04:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.3s +[May 4, 2026 - 10:04:11 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 4, 2026 - 10:04:12 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.550 +[May 4, 2026 - 10:04:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:17 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.478 +[May 4, 2026 - 10:04:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:22 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.095 +[May 4, 2026 - 10:04:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:27 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.249 +[May 4, 2026 - 10:04:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:32 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.070 +[May 4, 2026 - 10:04:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:37 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.079 +[May 4, 2026 - 10:04:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:42 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.076 +[May 4, 2026 - 10:04:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:47 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.482 +[May 4, 2026 - 10:04:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:52 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.217 +[May 4, 2026 - 10:04:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 10:04:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 10:04:57 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.153 +[May 4, 2026 - 10:04:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 700-------------------- +[May 4, 2026 - 10:04:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 10:05:14 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 700 +[May 4, 2026 - 10:05:14 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.145 (n=10) +[May 4, 2026 - 10:05:14 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 700 +[May 4, 2026 - 10:05:14 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700 +[May 4, 2026 - 10:05:50 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 700 +[May 4, 2026 - 10:05:50 | INFO | fastgen.trainer:run:232 ] Taking a 10 sec nap and exiting training. +[May 4, 2026 - 10:06:00 | INFO | fastgen.utils.distributed:clean_up:88 ] Distributed clean up. +[May 4, 2026 - 10:06:01 | INFO | __main__::49 ] Training finished. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/requirements.txt b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/wandb-metadata.json b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..058507657264bc47cb4afd5d9c33e1de5cf000c0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/wandb-metadata.json @@ -0,0 +1,71 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-03T23:10:11.016594Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769_no_reward.py", + "-", + "trainer.resume=True", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769_no_reward", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=700" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1175596597248" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "wwelpjm63fo275ghix3t7j87r0vw87x0" +} \ No newline at end of file diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/wandb-summary.json b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2459a4703d14a91173c536d7e352f02a88f453c2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/files/wandb-summary.json @@ -0,0 +1 @@ +{"profiler/backward_pass_time":10.131556924432516,"fake_score/total_params":1421383840,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"_step":700,"model/local_total_params":[4285560952,4283762808,4283762808,4283762808],"train/fake_score_loss":0.0113372802734375,"val0/sync_c_sample_8":5.217334270477295,"val0/sync_c_sample_0":6.550217628479004,"profiler/avg_forward_pass_time":11.963658344000578,"val0/sync_c_sample_1":5.477814674377441,"val0/sync_c_sample_5":7.078821182250977,"model/total_params":17136849376,"train/vsd_loss_weighted":0.45361328125,"val0/sync_c_sample_6":6.075653553009033,"_runtime":49956.81634984,"fake_score/trainable_params":1421383840,"optimizer/lr_fake_score":2e-06,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"profiler/optimizer_step_time":1.5581010431051254,"net/total_params":1421383840,"model/trainable_params":2842767680,"val0/sync_c_sample_2":5.095176696777344,"val0/generated":{"videos":[{"sha256":"8dad6b26232c7833b2a6e0eb9d260b4eb2fc2c3e71f6a58a799a0d56ea3ec749","size":214021,"path":"media/videos/val0/generated_700_8dad6b26232c7833b2a6.mp4","caption":"sync_c=6.550","_type":"video-file"},{"size":215913,"path":"media/videos/val0/generated_700_de88448d1dfb6875a6d0.mp4","caption":"sync_c=5.478","_type":"video-file","sha256":"de88448d1dfb6875a6d008c17358b947a6b5256e47e8b8a0e0ad904f48ae16e2"},{"caption":"sync_c=5.095","_type":"video-file","sha256":"0a9e7fd50394d8317c8bc55c367cf912a0f675c7c4403bfb00e6fb37947cbc15","size":205917,"path":"media/videos/val0/generated_700_0a9e7fd50394d8317c8b.mp4"},{"size":298771,"path":"media/videos/val0/generated_700_742edf4db049b9a7a78c.mp4","caption":"sync_c=4.249","_type":"video-file","sha256":"742edf4db049b9a7a78cab236fa5f79cd4b89f5ae7bf78bda169b3d55fb54687"},{"caption":"sync_c=4.070","_type":"video-file","sha256":"195f968c26d2d073200e7d141eed27288d6a8821148b8b6c246bc21e9171caf7","size":257491,"path":"media/videos/val0/generated_700_195f968c26d2d073200e.mp4"},{"path":"media/videos/val0/generated_700_3b5f4dc6836b99443638.mp4","caption":"sync_c=7.079","_type":"video-file","sha256":"3b5f4dc6836b994436382440658bf597e535e26ad49f28f4c72f6575fde916b4","size":185442},{"caption":"sync_c=6.076","_type":"video-file","sha256":"d5a22a2ad522cb8f20aad353e2b6ae96dc1a895b543edb553fd76468bf8586a0","size":299145,"path":"media/videos/val0/generated_700_d5a22a2ad522cb8f20aa.mp4"},{"sha256":"279df03ae1fa72af4482c0d8f27e9f6533243e3ffdb60761b9e3770b6d26ad78","size":200194,"path":"media/videos/val0/generated_700_279df03ae1fa72af4482.mp4","caption":"sync_c=4.482","_type":"video-file"},{"_type":"video-file","sha256":"c77651e7527fa1dcdf2c2d0d4cc4b28999261b7df746eb3810eadaf0b9a0fa87","size":237521,"path":"media/videos/val0/generated_700_c77651e7527fa1dcdf2c.mp4","caption":"sync_c=5.217"},{"sha256":"9f9787ff0ab4e25b8d0d6d060b74351396b141c82ed05730ac2cf434b0bf196f","size":245561,"path":"media/videos/val0/generated_700_9f9787ff0ab4e25b8d0d.mp4","caption":"sync_c=3.153","_type":"video-file"}],"captions":["sync_c=6.550","sync_c=5.478","sync_c=5.095","sync_c=4.249","sync_c=4.070","sync_c=7.079","sync_c=6.076","sync_c=4.482","sync_c=5.217","sync_c=3.153"],"_type":"videos","count":10},"train_media/data/real":{"path":"media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4","sha256":"2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae","size":190461,"_type":"video-file"},"train_media/student/generation":{"sha256":"54d7e0b7b8631130c0130dc593aca13885eead1f40bb955833b5027567d55eaa","size":215988,"_type":"video-file","path":"media/videos/train_media/student/generation_500_54d7e0b7b8631130c013.mp4"},"val0/sync_c_sample_4":4.070111274719238,"net/trainable_params":1421383840,"optimizer/grad_norm (model_key net)":0,"optimizer/non_finite_grads_count (model_key net)":0,"val0/sync_c_sample_3":4.24941349029541,"val0/sync_c_sample_9":3.152714729309082,"model/local_trainable_params":[711111248,710552144,710552144,710552144],"train/vsd_loss_unweighted":0.45361328125,"val0/sync_c_sample_7":4.4815778732299805,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"_wandb":{"runtime":49956},"net/local_total_params":[355555624,355276072,355276072,355276072],"profiler/avg_iteration_time":45.76494002342224,"optimizer/lr_net":2e-06,"train/total_loss":0.0113372802734375,"val0/reconstructed":{"_type":"videos","count":10,"videos":[{"size":182028,"path":"media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4","_type":"video-file","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327"},{"sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214","size":170922,"path":"media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4","_type":"video-file"},{"size":173236,"path":"media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4","_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8"},{"size":249372,"path":"media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4","_type":"video-file","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c"},{"size":190004,"path":"media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4","_type":"video-file","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117"},{"path":"media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4","_type":"video-file","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630},{"path":"media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4","_type":"video-file","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986},{"sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516,"path":"media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4","_type":"video-file"},{"sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001,"path":"media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4","_type":"video-file"},{"_type":"video-file","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490,"path":"media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4"}],"captions":false},"_timestamp":1.7778567142701461e+09,"train/gan_loss_gen":0,"profiler/data_loading_time":0.0008512828499078751,"optimizer/iteration":700,"train/gan_loss_disc":0,"val0/total_loss":0,"train/vsd_loss":0.45361328125,"val0/sync_c_mean":5.144883537292481} \ No newline at end of file diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug-core.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f38aab57942a2fc928acec91f9ff19f075dcbc57 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-04T08:10:11.139220965+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmptrentaye/port-3985335.txt","pid":3985335,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-04T08:10:11.139738729+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3985335} +{"time":"2026-05-04T08:10:11.139728875+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3985335-3990599-2782619148/socket","Net":"unix"}} +{"time":"2026-05-04T08:10:11.316624464+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-04T08:10:11.33224612+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lw2bm1l6","id":"1(@)"} +{"time":"2026-05-04T08:10:11.771606277+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lw2bm1l6","id":"1(@)"} +{"time":"2026-05-04T08:10:17.713311234+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6i0o3in7ghg5"} +{"time":"2026-05-04T10:06:01.242690915+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-04T10:06:01.242741522+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-04T10:06:01.242739426+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-04T10:06:01.242786664+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-04T10:06:01.242825401+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3985335-3990599-2782619148/socket","Net":"unix"}} +{"time":"2026-05-04T10:06:02.592506654+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-04T10:06:02.592529701+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-04T10:06:02.592537618+09:00","level":"INFO","msg":"server is closed"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug-internal.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7d9da8d01699b655a105b34d3cbed7c971bc118e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-04T08:10:11.332926177+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-04T08:10:11.770436324+09:00","level":"INFO","msg":"stream: created new stream","id":"lw2bm1l6"} +{"time":"2026-05-04T08:10:11.770484268+09:00","level":"INFO","msg":"handler: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T08:10:11.77160046+09:00","level":"INFO","msg":"stream: started","id":"lw2bm1l6"} +{"time":"2026-05-04T08:10:11.771618514+09:00","level":"INFO","msg":"writer: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T08:10:11.771631004+09:00","level":"INFO","msg":"sender: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T10:06:01.242751452+09:00","level":"INFO","msg":"stream: closing","id":"lw2bm1l6"} +{"time":"2026-05-04T10:06:02.302129532+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-04T10:06:02.591565787+09:00","level":"INFO","msg":"handler: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T10:06:02.592126222+09:00","level":"INFO","msg":"sender: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-04T10:06:02.592143864+09:00","level":"INFO","msg":"stream: closed","id":"lw2bm1l6"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5ae4c09fed3e52e66bb958ec23731fc9bc1b0157 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug.log @@ -0,0 +1,22 @@ +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_setup.py:_flush():81] Configure stats pid to 3985335 +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug.log +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/logs/debug-internal.log +2026-05-04 08:10:11,034 INFO MainThread:3985335 [wandb_init.py:init():844] calling init triggers +2026-05-04 08:10:11,035 INFO MainThread:3985335 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '700', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-04 08:10:11,035 INFO MainThread:3985335 [wandb_init.py:init():892] starting backend +2026-05-04 08:10:11,316 INFO MainThread:3985335 [wandb_init.py:init():895] sending inform_init request +2026-05-04 08:10:11,331 INFO MainThread:3985335 [wandb_init.py:init():903] backend started and connected +2026-05-04 08:10:11,335 INFO MainThread:3985335 [wandb_init.py:init():973] updated telemetry +2026-05-04 08:10:11,358 INFO MainThread:3985335 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-04 08:10:12,422 INFO MainThread:3985335 [wandb_init.py:init():1037] run resumed +2026-05-04 08:10:12,425 INFO MainThread:3985335 [wandb_init.py:init():1042] starting run threads in backend +2026-05-04 08:10:12,705 INFO MainThread:3985335 [wandb_run.py:_console_start():2524] atexit reg +2026-05-04 08:10:12,705 INFO MainThread:3985335 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-04 08:10:12,705 INFO MainThread:3985335 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-04 08:10:12,705 INFO MainThread:3985335 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-04 08:10:12,711 INFO MainThread:3985335 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-04 10:06:01,242 INFO wandb-AsyncioManager-main:3985335 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-04 10:06:01,243 INFO wandb-AsyncioManager-main:3985335 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/run-lw2bm1l6.wandb b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/run-lw2bm1l6.wandb new file mode 100644 index 0000000000000000000000000000000000000000..29f5974cb6726e91c007167530489b61c1fdae35 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260504_081011-lw2bm1l6/run-lw2bm1l6.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e33d16d4df0946b81f43e4f3321d6282aaa39d99ed8ac3c3be49f53e0638c80a +size 1556353 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..d5e5634df9b8c67d5e17cccf6769e509b27b3bf9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt @@ -0,0 +1 @@ +lw2bm1l6