diff --git a/.gitattributes b/.gitattributes index d7bb691dd32ec9d128c8142033276f433ba07f9a..93426213a3ee9480604afdeadb0c5038a671dfff 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10905,3 +10905,204 @@ FASTGEN_DF_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_stochastic_attn_ FASTGEN_DF_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_stochastic_attn_shift5/wandb/run-20260406_034020-k5eolny7/files/media/videos/val0/reconstructed_9500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text FASTGEN_DF_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_stochastic_attn_shift5/wandb/run-20260406_034020-k5eolny7/files/output.log filter=lfs diff=lfs merge=lfs -text FASTGEN_DF_OUTPUT/OmniAvatar-FastGen/omniavatar_df/df_4gpu_bs16_stochastic_attn_shift5/wandb/run-20260406_034020-k5eolny7/run-k5eolny7.wandb filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_100_5f29ddd650acb76cf120.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_200_943d779064d5cfaed0f3.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_300_4ab5262a7b1c0d68aaeb.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_400_da09172e9304b45512f0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_500_6a6b80f57451f4c65363.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_3f4b4b4b1f7994005691.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_60678e42d5e07d910455.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_63236b93692d9d9e51ea.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_772250c2a870faf7a028.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_78916ece5cfeda35951f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_b6281b242a558e060d07.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_be045955790e2bafed38.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_cfb620745c75deeb421f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_e849a42a36eefa83637a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_ed9bb4b97fd7ff56d04a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_0c507e08b87db5dadd85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1a2c6afab56212e63ea3.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1af0c947a6ca7e10204a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1be645cb586cdcd86592.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_3bff22040682ad02a71d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_45b97079bc1142461e90.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_63c99e843c95db59d2ff.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_9fdff62b734a70534467.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_abccde721f4fe693ad51.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_f8200bfce19fbbbea6f9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_08f28a9516c32be2ced2.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_14e28af9362159fa3209.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_1ab0d4fea92c6cd17c67.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_2d845c09834840a31676.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_3d212821be27a78aaa54.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_425fa942f39d9fe58fc9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_4bbc26f982c0519df9bb.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_540eca29fd8610e1c3c3.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_940d532a3e7d1080a806.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_cce916972215c524ff32.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_32c745f139009d16c969.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_71598f1db28d957e3ad9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_82c31f2597df9e163331.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_87f1cdc469b1c4887352.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_d7a0f4d9a1a1801f46fe.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_d811014af75e5e746ab0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_da44218d10aa75385964.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_ed4d62bf1ee02c397a9d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_f07e2686e93beda72196.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_f0e920a6e85db5765813.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_1ef5e9e74ce125bf5d98.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_23de0a821d0f0295c056.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_24d7fdfa7352306340de.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_5747e2fdde178d76596a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_57973f55e8d100462b9d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_5ea3dd7b201229042f2a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_8e523871bad14b557cb0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_a124c4cd33f0a122841f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_b5ddbe7301ffe322a095.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_e36b7bc1deaaf550e6d8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_02c1aae0e691f96ea5ad.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_2dc2540d510c8648dc00.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_3af3ff96013a3f3a3e81.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_86f8df0cf9c178a9bc20.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_8ae4464b03faf92d8fcc.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_90413783c59d55141104.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_9db48cbc78876e078e80.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_c2ea37b78886ecef23af.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_ea52c20a941a74fc52e8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_fc14a16bac32611ba42f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/run-c4sg4krp.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c6d913edf43b9c9cb544596b15c1ba68f373c2d2 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc04be4a69c5d6b58fc72caf3a152184e0b567d9b3ab626e9ac5be2637d59d5c +size 635510 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..51396d7ffe85a7ed7f1a104a2fa9a224c276f02c --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:336174aa28431e99201ff2b445589083164997f4d998ddd1705d751fa9681348 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bbb2d5182e03652e49e18a2514fe1e7d216c4870 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bcf86a38217f136309bdb5257974995025fe9b808985e7be2cdbcdde0e2e0bb +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7e17082c0726abb1a59457406a28b0767da494c2 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4d3b571fe51ead8045689f2dbe4450c3f96f6bc1ebdd3694bd0e0714a595f90 +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..687a6ebace2e561ead8441bc34c8e01ee33e6d1c --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a4dff7bc375fb59b084cc2f2a1168c7effdeeba1a3594ea2c717eefd4fb10b5 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ae80a9bb5a3ca669f8b41bd35824306f2d996cbb --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9afd81e1716bf358716562c1ab79c3d23f46029852fce1c382bb535c8dd70c +size 635503 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9f49d45e95b6049d6fb0b7f98d0627904d5e54d1 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6aafe37fdd512ebbe8b03865d1039f4ca66e55b5aa0e7490c076c9dfbda0d86 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2354dc3fe2417f36a409748f96f2f4870bd59fa6 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eb6dce251705e5000ed83c29098226e342dc1760fe8713ccf520f0f033cf368 +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..83f9e1f406f9394693a9b45f5d9a688eac04459d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:787a316f45b69d236fed8b8707db374ce2aac3e9af74ca298a520841ff4f0831 +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..87690a3f48e6293d190b99aa4a22f0f07ba99147 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3468ebb3daf631514ac4b2cdcf89262d19284670b0a64c6a87f2b73c09fc9192 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.pth b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f637e0f6792d8ae1960e0e448573321eacbddc4 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9e7bf7b483afea9b5f5fc9f6d1a8368eea2246e5e5404c08b9fba8acdc0064 +size 1901 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..73b22f8ee6c23c29f9cb7daf8d2a31eeede45c16 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64a496ef55a80f9220b46c437640c2198da8e440cd5d175ba6f918e2a75b48d8 +size 635510 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..af297fb15939555446d8ac06cd4252c9279ce5e6 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07c238fc68230a28bf33d19ab7e99537e9d1c88130c68728bd7864a730dfbf44 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6b5db475bd087f8e66e421a0c0d784d041f70c7a --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79a8db35beff57646878606b29edf64813e66768e984043a21c89d3af9ed2cd3 +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9ed5c439dd39736017def7f6292f843f18c82391 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e0a98cc66dabef067f5a6a4185a34b1a99c55a307b50166fcab10f02b571c6a +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b045e402dea5d95c899bf57b85bb5b895fb25b3b --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bc48f727bcc79804620bc3b38f8cffcb1d2960522d273521caced4d5fcaa324 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..a556219127abb1a422d81f31286a9dd9ab3b2ba9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2026d6bcee4493f55a1ae71239e2d358f1d9dee91505be9a440bda4f8975182d +size 635503 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8f2e144f25d7e388f05320b90517b006cb26e74b --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0a85c3d05b2190cbd988c25ef3a4353d089af8a170b608eb152c3ca6f02b4c9 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..168e6bdc2a454e7643ce4f0dd864506b2a7fa62e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e452b694fe1c0ac7b2a7d958c77f8927247fa113e5a83fdc2008cca8be8f2244 +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..90c4d29104b7d231b6c44de8fbd040cfd9900511 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d3d6d8a8bea35c02598bd23d0da228c672ece1eee70527c684e6b407241d072 +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..29fd5e2ed8402484a7a6fee633fc88c50139d1e3 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d6e746046dd081345fda781d6d14dd47b7337926ab472a6877fc4fada20f5f2 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.pth b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.pth new file mode 100644 index 0000000000000000000000000000000000000000..94689c6d88777305ec14be3e82dcf676c28c81a1 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d49c29dddf4cff9abe84c331dfad17799824bf4158c145f77812c5430228f1 +size 1901 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..737e90fd64ec4487efac9b0396b9f1cf6eed232e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7abdc74389b48d1b41cc6340eaff49d086be33e18e7f3a5b151466cf57b34019 +size 635510 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9bc6ccdd4c9bf0a7546ce7b4df5a71d1794ae555 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd7f5ca6ff5896eb9e15bb9ad1f5f6c4a16690e8016d6adf378b19a1e2689ba3 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..44d8b35178710c91014d1a8e5c72d14eeef97ec6 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d04fbce4efc3505d5fb0af2e5c28ec708abcebcc74fc62794473c4b6da12621f +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..39ca366b784188cd92b7c2a73721645f20f07c35 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c235be9c22c77278534b62344440102a9465dba4292cc06f0699c2dda4bc12d8 +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f3fec95532aa8e52619a46ec3e1e473c44bb4718 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca57b6cbf79caad59b00e9016322b18770711aba54170bfdd59d01231f4115e +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ef6744e09272cafde1918e48f7b4f8dd9158fbfd --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91898efeb5d28de4827f31c46cfc06db2cb50d96431f9cdaf903abd3d0fd7f1d +size 635503 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9642c010103e62cb9f678559c4bd8fc3ff8f5d36 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17d6ae8923b599ffa7cd396fb0c9a59417be298115a6e0a1299225b712e8d72 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3a3ef85db8580c1ff29614ff7b860e3ad7c46d1a --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cc1d13c431665b935f7b4798eed4afa9b3a92627aac807c0d6552e527ba94ef +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e3a1cb0bd374087676dd238f823d1cb128603cfa --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:852afd1d47119b3be887d79039b4f2878dda65dae3d65966257f341e9ff449fb +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..76352bbc7de3b4196cf47b25adb2b2242a9a30c4 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:360c0cca9bec718d13f63a658b5ea00f10851ab852c295fb834f4fb27ccee165 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.pth b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.pth new file mode 100644 index 0000000000000000000000000000000000000000..36cdfef63708c11e993e5164fd9859eb64d6d33e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463104a57d6bfde142e5cd01a059afdb8b99dad40c73e04a236502ee34fb4d66 +size 1901 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..b0a11216d9ee41d644fe31d0fb02fbdb8738d1a2 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1bbe039576fa042355af8e6f8817036fedfa91a2a3421b4dd7d375b16a80930 +size 635510 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..56e0e7604be5657766071a1777a1d02e68e609d1 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bed52dbcfe934154a903932ee98f3296f0b07c378bbbe7a815095df28623cc2f +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..de16d10c7c01d6bc023fbcb3ed18a62af9b138f0 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c32fd3f6a7d18852a8c41df9542500b7fd76bb53464bce5fe1b6dd64e183b7bd +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e2f45e8dca505de04ef5cce690ce3e6bd5f4b244 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1facfca5632f4136d8d81f6b9b271499dc5b7a80fd02be3a12591219c0fa7fc4 +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6c38c844bec3166a472dbee7861cfff4708882c0 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00224befc70b0f66b2f075c7b3eda23ecd467fc34f5b66bd5210eb13ea48e3a4 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..392f87df1756b8237782228bb568c20ec356f20c --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb3b149b9b82840da3aa9de7c25e468193ab0528714efce094f58967f37d1097 +size 635503 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7d078c50a63a7895d1838132cb2bfa27f7b787fa --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edaf72f4050f0f00c5faf54a95ec5a89d6d5a1b75e7b8ef1828ad49a6d3d1a62 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b5963dbc8a080006916455e68986b9545431faf4 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de0d6194a8da09c799d12b985b9b92d750b02e145e53d68e1ec26f1702986b6c +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7ff674aaa9c6c4b72026a7b731cb26762f842b32 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc702c562222f0c9124ce4db70a80779b8b1b49abbfd61b5825d51fc932c5ae6 +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7f52b7063742f7f09446c77669f8c6bed621d3a6 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:993e574dd81f176060802f5ff56fabd2b7a2063f31c6cb1a6285214077943fb5 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.pth b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.pth new file mode 100644 index 0000000000000000000000000000000000000000..0deb1499fd51197d9815f644a93381fbe0678161 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31545bbbd2979dddf082789dbf999d9324dc9febf1b016304ddc713ede429c3e +size 1901 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..b3bd489170087f900446d1ef999f12224c389baf --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55472db04fb70b18bcb46a94e1371928f08010cc4e3843de9e548c66952d9508 +size 635510 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e458ff1fb8b31b6fb7e40e49037615857ea44855 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c43076efe1b1e41e3a8d8e676a758462f2d71879452f0a1e3458b5aec6ba7360 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..078d6133e95566e735e2a23aa241e20f23a1a2cd --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6163b80c8ed1e05d81b42b76453a49d1c55ce1320284fc06a3ff7ba6c72c3f4b +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e74bb950d4432b754591890be68aa27a5352493e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb828121e823230f2f1d237ae46e7acb8bf467c3302cde6c7fcd55b1d932918c +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e674388f5305aeb971ea845b61ec654554a85238 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:471eeefc3ae7976697effebe3266fbea5609ca8747dc30e7cc6372d4f7e36085 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..337b691d9364a4c555df43ffa78b0b3ad8b4c74e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fb5c22fe2577c92633270410807d8b18774b4d954f807a78396b95c67c4fb84 +size 635503 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..84fd7a4051ad32d58de53d7dd1519cf01d664361 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:866c6bccdf1052db456394e2141c3a4277033f5e020bff7e139cac5eeaff1438 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9e59e1bb51f734a80a113d94e6df930598c99471 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:181d0a33718c2f75a2af2f8e8de9dd40bf29f2e733f5bc5adbd41d2f975f4edf +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fedd9c56e9397e2ac49424130b68ed96b9fae38b --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51f15a3feb48087cc6ce5b487459c805898d76c9ce8fc36ef4ee9dc7443d3a9f +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..47c2480af44cfe7469a7b72ac71e92c4e5e8ffed --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e71d4b3aea0315e4fabb7d72611d3f1bb9d9c5d2884b578ba94508900951e00d +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.pth b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..df0fb483027e40ac33d6a36f4b38fcfd244d6a44 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2110994f1f90ec15f3f831251e70d29fe1b83f8a35bb969069c4ac3f211c6d +size 1901 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..98fdf56d6a1994b58ffe890bfd228c70a9c3aac0 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d96a35d08dffe0e797685cb1875759551b375e2e869e5f728feb249b789b17e +size 635510 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6c0aa775e68b4583f36f2b71c733fba7f2203623 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6373ef3d419583963ce9399fefa4eb80e3b7110dd2233549534a5fe4a4e98735 +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..cde0c0e73386240064bc89675708436c514b8b04 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06fbcbdd8502ae3518b5e716d5770e3dc601a1228c5ff6451f570e7130136efc +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..679aeec6858107da1d6677c6a857355f74b7afc2 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:279505ad5a0a580c799a12bfe31f21511522dc47cbb95aeb7b7031558cd5c4e2 +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..65a773094eb0eade48c4b5830ee47f151db8e59d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39a6dbb6112ccbcd779db174527495b43fd1ef6d6b6a2b2d924b5c10659a8c51 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d67fd354c62c6535317fa41fffbbb1c3f9471b95 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2d0895d0de5b452fbd9e2f902e07880bd07798d774bdae3ff94eb40b347f6d2 +size 1722874 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7fcc336ae2b1e5d6a81101513b4efa9944abedce --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63d2352b714f3d8d4f71efc9e681fd0d8932d571154a1cee83b1d20edb19908e +size 2847149355 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a25094bac21675453bc9b120e48a900512918a21 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5630903dfc44613093ca9fd9ea899bd016a664ac83b32081294f82d2a75e2c8 +size 2845267132 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b1742d964a28a4546d55213b037c0778c28a84cc --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15bd37b6577222f904fc098422de09a6e9463d44f774a1201b3dd94555fed6bc +size 2845352958 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9d360001c0052edd078254443ab3434f4f51df5c --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:904d1ab30e8484774a264fa4309a5d58a2292c3e71b16b6de45210f544accfeb +size 2845316241 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d9eb470ea5f7fdd425d4493ae5f6520e924be440 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:220623aea9f3295aae4cdd8ac59139779390eaa1b852fcba5f465ad7b327a1a8 +size 635503 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..69a4bfb79517c190e7b9eb77593626cccdbec0b2 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41421ca14530fa88b37678617c10146d5852ccd34ec639ce8006c92e789afa2b +size 1423573889 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6a1a7d3603419f65093ab2e7a0bdd4c936f7d34d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09c92ee0b080d1e38caf16ce5d1103b2972fc706f858847970b5ccbfec230bc7 +size 1422406794 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4f50cd93b6ba06d7aa7c22fc5782f5673da94d4b --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:235f1a7509ff2b057629cb07e14b00dbecca1d88b551317e466aab78280d9174 +size 1422431002 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..82510dbbeef86d05a18a0a67fc8c067d8c6f4c97 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ba61a861bef1b6702b086b9265a5a88ab073f58fdc253075637ffee3b81e7e4 +size 1422429489 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/.metadata b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ae5bcbe1c46e9deca449fd22b7615248fb79166d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f52554d88ced0d1736e0a6d0f21d637c1513b94f6bbe8b07222c873f2d2decd +size 1722717 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__0_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ab5ae3a28751f202a85ed145d3e8ebaa6a6df2ce --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98b42131488050d6dedf20e269a6fad220bb7cdab8501cbc8dfc541644d86ede +size 2847149355 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__1_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2530ba8c3c090800b8d887a8f946ba066b31ee9f --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a96c1937b149d493dab7afcef037fce8c2912d2a6b54eaa1fa73b846a14286f +size 2845267132 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__2_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..61ff3343a9ac9c2d75e796cbe432885d79bdafc5 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6482c5e8fe7281f0323c8a1a69ae7e88f5ffd90487d7e22d705b48b2c3e91ed3 +size 2845352958 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__3_0.distcp b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0c5d3784e3e9d0a1d8177385357a8989911b4090 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:005d3b76fdad4ea3a3d2f55ccad515fbb0e79ff39a6e1c68e6d0545d46a925fd +size 2845316241 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.pth b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.pth new file mode 100644 index 0000000000000000000000000000000000000000..97cb41dfed21f440317a176863edebc119d1a324 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c1c25fd6c3ea2819196abcf7cba095dde0d86545b36426cd731e4feee3caa8 +size 1901 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/config.yaml b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a57517788f9f679d1cd9006ce99f931fb3f1ae0 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_2step_t937_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'False' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '0' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.937' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '600' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug-internal.log b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..cf8097dd2c52fc2162d709c1a3b71a8647371d3d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-05-06T19:10:16.799200194+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-06T19:10:17.443315611+09:00","level":"INFO","msg":"stream: created new stream","id":"c4sg4krp"} +{"time":"2026-05-06T19:10:17.444248699+09:00","level":"INFO","msg":"handler: started","stream_id":"c4sg4krp"} +{"time":"2026-05-06T19:10:17.444325118+09:00","level":"INFO","msg":"stream: started","id":"c4sg4krp"} +{"time":"2026-05-06T19:10:17.444355369+09:00","level":"INFO","msg":"writer: started","stream_id":"c4sg4krp"} +{"time":"2026-05-06T19:10:17.444354811+09:00","level":"INFO","msg":"sender: started","stream_id":"c4sg4krp"} +{"time":"2026-05-06T22:52:10.007450117+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/paulhcho/OmniAvatar-FastGen/c4sg4krp/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-07T06:55:05.903830281+09:00","level":"INFO","msg":"stream: closing","id":"c4sg4krp"} +{"time":"2026-05-07T06:55:07.347145243+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-07T06:55:07.720260545+09:00","level":"INFO","msg":"handler: closed","stream_id":"c4sg4krp"} +{"time":"2026-05-07T06:55:07.720330233+09:00","level":"INFO","msg":"sender: closed","stream_id":"c4sg4krp"} +{"time":"2026-05-07T06:55:07.720346042+09:00","level":"INFO","msg":"stream: closed","id":"c4sg4krp"} diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug.log b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3fe01544b66f4bcae1e9d87c1e4ca4c5c53c3ca1 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug.log @@ -0,0 +1,21 @@ +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_setup.py:_flush():81] Configure stats pid to 3167398 +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug.log +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug-internal.log +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:init():844] calling init triggers +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_2step_t937_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '600', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:init():892] starting backend +2026-05-06 19:10:16,780 INFO MainThread:3167398 [wandb_init.py:init():895] sending inform_init request +2026-05-06 19:10:16,794 INFO MainThread:3167398 [wandb_init.py:init():903] backend started and connected +2026-05-06 19:10:16,799 INFO MainThread:3167398 [wandb_init.py:init():973] updated telemetry +2026-05-06 19:10:16,819 INFO MainThread:3167398 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-06 19:10:18,404 INFO MainThread:3167398 [wandb_init.py:init():1042] starting run threads in backend +2026-05-06 19:10:18,680 INFO MainThread:3167398 [wandb_run.py:_console_start():2524] atexit reg +2026-05-06 19:10:18,680 INFO MainThread:3167398 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-06 19:10:18,680 INFO MainThread:3167398 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-06 19:10:18,680 INFO MainThread:3167398 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-06 19:10:18,686 INFO MainThread:3167398 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-07 06:55:05,903 INFO wandb-AsyncioManager-main:3167398 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-07 06:55:05,903 INFO wandb-AsyncioManager-main:3167398 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/config.yaml b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edbce9dc3c40e78cfe7c31bb94c85e93a302c013 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/config.yaml @@ -0,0 +1,388 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + mtg4sjqss4j2qo4zq4jrh08h43c2gge7: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_2step_t937_no_reward.py + - '-' + - trainer.resume=False + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_2step_t937_no_reward + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1165820178432" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward + startedAt: "2026-05-06T10:10:16.306844Z" + writerId: mtg4sjqss4j2qo4zq4jrh08h43c2gge7 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_2step_t937_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "False" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "0" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.937" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "600" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9684c7bde9ed18086d39e7a9880b642c505b3947 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112c0905448a821974f0bf0a94d9107473bb02b60e0ebe1e75a57d309670facb +size 219311 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e38036c10e966c2c82fa717b29431a1ff88e1ae --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae +size 190461 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_100_5f29ddd650acb76cf120.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_100_5f29ddd650acb76cf120.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b321bf454e35a8a12d1f71e68538208b65b016f --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_100_5f29ddd650acb76cf120.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f29ddd650acb76cf120eb9ec98edc575eb211a35cb0fbac9e42d12d5418d817 +size 184738 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_200_943d779064d5cfaed0f3.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_200_943d779064d5cfaed0f3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..573eb847a94bc4b900cca371e08cc4e93783299b --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_200_943d779064d5cfaed0f3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:943d779064d5cfaed0f3f44174a8f11ed2728953d9a520efee4d25fe5bbf9429 +size 187088 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_300_4ab5262a7b1c0d68aaeb.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_300_4ab5262a7b1c0d68aaeb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ba5b95182d7bc4e1908ae32a756de4a22899777 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_300_4ab5262a7b1c0d68aaeb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ab5262a7b1c0d68aaebe6894d366bd64406dca5dd04ae9e76427fb801f88f82 +size 208017 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_400_da09172e9304b45512f0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_400_da09172e9304b45512f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a30300188edd82a40a208a13db413e9b6ee3dcba --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_400_da09172e9304b45512f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da09172e9304b45512f0424c330cc6079816ff67fb197b11a8f74180f6587f25 +size 223195 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_500_6a6b80f57451f4c65363.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_500_6a6b80f57451f4c65363.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ee8d1f67cad78a5959a4ae78db506c668ca9936d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/train_media/student/generation_500_6a6b80f57451f4c65363.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a6b80f57451f4c65363131082571e6a60ea0a82948c057269716527cf91b416 +size 198813 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_3f4b4b4b1f7994005691.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_3f4b4b4b1f7994005691.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..745b0f17f9e47085cee5b6a3b612e3182d92fc9a --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_3f4b4b4b1f7994005691.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f4b4b4b1f7994005691f103f888418f7c46417606ec4530b9fb9c17f4af2143 +size 187134 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_60678e42d5e07d910455.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_60678e42d5e07d910455.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cad1f72cffcbfd97849faabd35f703c6bef19a2d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_60678e42d5e07d910455.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60678e42d5e07d910455ba0aa5efc6e2bf1df194765ed103e974fbf8aa3e3225 +size 240765 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_63236b93692d9d9e51ea.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_63236b93692d9d9e51ea.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9c7049d3322170140b6f37a795572d0126c9a14 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_63236b93692d9d9e51ea.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63236b93692d9d9e51ea533070e6b592fc298c6c4cd54d41219df62706750035 +size 208763 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_772250c2a870faf7a028.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_772250c2a870faf7a028.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2d01703d4b567c737a3d86cb62c5141ad2bb14ba --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_772250c2a870faf7a028.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772250c2a870faf7a0282c7daf20617cd564abf78a8cc71b9b21c83f927f1e50 +size 209864 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_78916ece5cfeda35951f.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_78916ece5cfeda35951f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dbadffce9e7ade178c75d209a5b9a3e4228d52e9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_78916ece5cfeda35951f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78916ece5cfeda35951fe71cc40db9dffbed96eb170bf980bc3756f7d4cdec72 +size 258959 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_b6281b242a558e060d07.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_b6281b242a558e060d07.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8fbb28f485c671f1a9b796af32a69e2e8ab0f420 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_b6281b242a558e060d07.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6281b242a558e060d07a412bd962e49356334d5710d3726d5f68b038883e204 +size 180265 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_be045955790e2bafed38.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_be045955790e2bafed38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..47ebb92eee39b72fefd070dfd91bc4e6fa5f8b56 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_be045955790e2bafed38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be045955790e2bafed38d06a6517749c4f170727243f6136114398bbe5deb642 +size 214798 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_cfb620745c75deeb421f.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_cfb620745c75deeb421f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24b8660a624bc5e74d9f567e73de39e0575c427a --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_cfb620745c75deeb421f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfb620745c75deeb421f1cec3292f5c691605c34aec03682fa862bcb1f950e7d +size 159174 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_e849a42a36eefa83637a.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_e849a42a36eefa83637a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9384c2ef7ab359c746cac4847a19a1746f9e90de --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_e849a42a36eefa83637a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e849a42a36eefa83637a166abf5e1b2c7dded15084c88a1b78ac4df89847849e +size 178208 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_ed9bb4b97fd7ff56d04a.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_ed9bb4b97fd7ff56d04a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3f8d4ce0ce927fc4a83e7992d027ab20593f320e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_100_ed9bb4b97fd7ff56d04a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed9bb4b97fd7ff56d04ab194bab0e1504acfd9439893d160e683e917ef169fe3 +size 168743 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_0c507e08b87db5dadd85.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_0c507e08b87db5dadd85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8af7476d8b458e88cea8d0cf28b75e004a784b65 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_0c507e08b87db5dadd85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c507e08b87db5dadd8593bf1df1e465e8f2fdcac02cab689ea6b8ecf3f0a8ae +size 250009 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1a2c6afab56212e63ea3.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1a2c6afab56212e63ea3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c453615c2a9ec40606a34964557a751fade5a9cd --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1a2c6afab56212e63ea3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a2c6afab56212e63ea37658b2742b3ca5c676b92e73aed20d9edf48ac3da9aa +size 216669 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1af0c947a6ca7e10204a.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1af0c947a6ca7e10204a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5656da5f55fcfc1eb818bb3124ad2880e22b872 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1af0c947a6ca7e10204a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1af0c947a6ca7e10204a6547e2499ecafeb99ef9d7aacebc6bebccc7ef92d81d +size 214156 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1be645cb586cdcd86592.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1be645cb586cdcd86592.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f98681e126824ece2f110014b491dfbd2ff3cb3f --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_1be645cb586cdcd86592.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1be645cb586cdcd86592bba99cbd899dc07068d1e77fb0318276c3ba8ea47921 +size 220849 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_3bff22040682ad02a71d.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_3bff22040682ad02a71d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0247980ee10282c2f6db62ddb2e3de916af6b8ff --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_3bff22040682ad02a71d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bff22040682ad02a71d91e923237ab836f03800655987173ddb94b68222b92a +size 266402 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_45b97079bc1142461e90.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_45b97079bc1142461e90.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3f57a254a5535f148084655fd08e449e4ee31194 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_45b97079bc1142461e90.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45b97079bc1142461e9058e9d7e68b1606c979f48edd78a3b0ee522afa83fb17 +size 179464 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_63c99e843c95db59d2ff.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_63c99e843c95db59d2ff.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..10d7df0a66a145ad65fb172e2a236bd4d9825c1a --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_63c99e843c95db59d2ff.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c99e843c95db59d2ff5caec5fc772066d7ffc85a40541923276bcbc57dfb00 +size 174187 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_9fdff62b734a70534467.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_9fdff62b734a70534467.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33de7a13bb19dfdb2aee08c2b87e66471eec5651 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_9fdff62b734a70534467.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fdff62b734a70534467196e60d692b20a25184f808915694ff09e99769567ae +size 165653 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_abccde721f4fe693ad51.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_abccde721f4fe693ad51.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..880680c6bf4eab9d901ed88e43d6f933fa410983 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_abccde721f4fe693ad51.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abccde721f4fe693ad5184f06587dadfc4a6586d77282d75811fe79120c65273 +size 186627 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_f8200bfce19fbbbea6f9.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_f8200bfce19fbbbea6f9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33150b00be3b8b769ab1eba4c50fd41b04361ef2 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_200_f8200bfce19fbbbea6f9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8200bfce19fbbbea6f977cb34db02418ff0b24965570b6b011ec09b6e3abcb0 +size 190698 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_08f28a9516c32be2ced2.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_08f28a9516c32be2ced2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..225c886b94b32c6fad0ff8b7349c9be302594777 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_08f28a9516c32be2ced2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08f28a9516c32be2ced2467d11e774dd0acfd6c51291b908d6b478e214561b96 +size 265061 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_14e28af9362159fa3209.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_14e28af9362159fa3209.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31b98717effb1f3a7b436ed9f90c3e4864e84944 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_14e28af9362159fa3209.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14e28af9362159fa32092502c9b00409c805f895d92bb8f2560e7feeb781cd9d +size 227512 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_1ab0d4fea92c6cd17c67.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_1ab0d4fea92c6cd17c67.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92d2bff55cbd4d1497d6a73af40a7135add50c88 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_1ab0d4fea92c6cd17c67.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ab0d4fea92c6cd17c676d671c97aece92c52b9cbbd535aa4816fec839fcf018 +size 232827 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_2d845c09834840a31676.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_2d845c09834840a31676.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7523c5debc0aa49a1d35c16ad3c47427343c21bf --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_2d845c09834840a31676.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d845c09834840a31676de2adf7ee6e691005bdcae8a62ab7d9a225335515b60 +size 191300 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_3d212821be27a78aaa54.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_3d212821be27a78aaa54.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cc878aaf899cd3bba54be32da18904a74f93d6cc --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_3d212821be27a78aaa54.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d212821be27a78aaa546c8838c32011f788964e1d6336ef1a3c7a8c3b9a2af7 +size 223838 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_425fa942f39d9fe58fc9.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_425fa942f39d9fe58fc9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..967d4919f91a0d95de2056a3ab49562089a692b5 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_425fa942f39d9fe58fc9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:425fa942f39d9fe58fc96f6302f40265556eb31183f670d19eeadc6bdbeef35d +size 200746 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_4bbc26f982c0519df9bb.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_4bbc26f982c0519df9bb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d326660806517c24c1fa487485c1ad2d2a05de16 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_4bbc26f982c0519df9bb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bbc26f982c0519df9bb06e1fbcbeeaa2c8f8ae4ffc1e97ec823eb2571391d0d +size 281932 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_540eca29fd8610e1c3c3.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_540eca29fd8610e1c3c3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f28a2bf4960a368e24398528e1ecedf45d2b29f8 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_540eca29fd8610e1c3c3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:540eca29fd8610e1c3c3b888158f59e4545dda16fc7e5b21751a62dfd8779ab1 +size 184337 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_940d532a3e7d1080a806.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_940d532a3e7d1080a806.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6797e31da67af32326ba41879c0feb0452c5b6e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_940d532a3e7d1080a806.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:940d532a3e7d1080a806796179d9666003430b80cf4f44503e602d99717aae9b +size 173109 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_cce916972215c524ff32.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_cce916972215c524ff32.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9f2b78d4d2a045b8cb6cae431f606ccda8a6a7e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_300_cce916972215c524ff32.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cce916972215c524ff32e486d18c0ec995a1d7c3a7c6f66841214408cae9db31 +size 198856 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_32c745f139009d16c969.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_32c745f139009d16c969.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f242e9d1bbf754786de9f764c426bfc5fad427f4 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_32c745f139009d16c969.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32c745f139009d16c969eed27348532763931e9c6311954761484101d3adbece +size 219344 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_71598f1db28d957e3ad9.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_71598f1db28d957e3ad9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05f224e9173b06510dbbd4455c8ebc1d95003238 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_71598f1db28d957e3ad9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71598f1db28d957e3ad92522c58838554f9bd79a29ae678865ed79770441c832 +size 171269 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_82c31f2597df9e163331.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_82c31f2597df9e163331.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cf7a78a9e9711551b5daefa8be9647af5413d0d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_82c31f2597df9e163331.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82c31f2597df9e163331d93b1ef9c5908e93b818da5df43184da37f3bc18e8d3 +size 181445 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_87f1cdc469b1c4887352.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_87f1cdc469b1c4887352.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..96b79f36ebd9034b6a116bed7b0d1673e4a54e64 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_87f1cdc469b1c4887352.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87f1cdc469b1c4887352aa511d6ff6aac7037470aaf61505a964ed92c444b09e +size 189731 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_d7a0f4d9a1a1801f46fe.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_d7a0f4d9a1a1801f46fe.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..db4189b08cdb3385c1d4ed93e119418155d941e9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_d7a0f4d9a1a1801f46fe.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7a0f4d9a1a1801f46fe853b3b2279cbdccd5a70843846f206f9d4725112c715 +size 195716 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_d811014af75e5e746ab0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_d811014af75e5e746ab0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2b613e33034bef9495745fa883940f9722a0e25f --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_d811014af75e5e746ab0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d811014af75e5e746ab0992879625dd152df158f36f6a8b632562806e6813d6a +size 263509 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_da44218d10aa75385964.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_da44218d10aa75385964.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ac33f95bc1368980b186c75c4d3546a25133882 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_da44218d10aa75385964.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da44218d10aa7538596469646d7ac0bfec1bca5763d779a8f6681f3f26da0dd2 +size 197114 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_ed4d62bf1ee02c397a9d.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_ed4d62bf1ee02c397a9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08ba9ba7edda2ae25811a2358163f76a302f18d5 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_ed4d62bf1ee02c397a9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed4d62bf1ee02c397a9d6b689af400d468bfe0d2a44f38df41ba803b189704ec +size 273717 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_f07e2686e93beda72196.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_f07e2686e93beda72196.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7047d29de618fc5618cf3b53aa05e6bf6c008182 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_f07e2686e93beda72196.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f07e2686e93beda721969f28d9f34da2e77e2db942e147148f2b4c77d4d16103 +size 223894 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_f0e920a6e85db5765813.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_f0e920a6e85db5765813.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..59ed6b1db785aa701423148879cd25c4cf1059b6 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_400_f0e920a6e85db5765813.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0e920a6e85db576581322276b652013494a3e3c430f84d4834c34e10d5ad196 +size 229445 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_1ef5e9e74ce125bf5d98.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_1ef5e9e74ce125bf5d98.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8a6dac3be1fddcba937c51a60332972494104a6a --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_1ef5e9e74ce125bf5d98.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef5e9e74ce125bf5d98e4c49642d88c38b64ccf31bb881d530c6d94f86f3ff3 +size 183984 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_23de0a821d0f0295c056.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_23de0a821d0f0295c056.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39aee913ee8ae8a346227af5c9b38f8e567edf47 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_23de0a821d0f0295c056.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23de0a821d0f0295c056f645174248183c952c264230eb1f73b2e733533f8e68 +size 200754 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_24d7fdfa7352306340de.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_24d7fdfa7352306340de.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..55ca926664878caca0a5202a258cba071dfda8de --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_24d7fdfa7352306340de.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24d7fdfa7352306340defe58d3a2f9ea6d3612e5899bc88463202b81eedde0ae +size 279153 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_5747e2fdde178d76596a.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_5747e2fdde178d76596a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d089b294ce029c687cd39bcd3faced9ac6f81dc9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_5747e2fdde178d76596a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5747e2fdde178d76596a728c9b3a24b0113fe2f7e48d237827bbb4b3ed62a6c1 +size 233336 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_57973f55e8d100462b9d.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_57973f55e8d100462b9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f982a26fd01eade4b545de17238d74e1010138b --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_57973f55e8d100462b9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57973f55e8d100462b9dcf1ca785806adf9368b9a777f2e47f07424c1b0d2dba +size 202098 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_5ea3dd7b201229042f2a.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_5ea3dd7b201229042f2a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78b0a79cde30de30b0fa86cac44935e3e932bf78 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_5ea3dd7b201229042f2a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ea3dd7b201229042f2af4ee218392eda8ce54ea27d861247d49a95ace2be06b +size 176838 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_8e523871bad14b557cb0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_8e523871bad14b557cb0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f30689b7b1ac8381e09d4005d53de822c87162a8 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_8e523871bad14b557cb0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e523871bad14b557cb04fb296a7b66ce0637b92e677d6b59fb3fb51d3ffbb19 +size 222993 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_a124c4cd33f0a122841f.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_a124c4cd33f0a122841f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c13371b1b17b4fcd53826074368ed371fa880846 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_a124c4cd33f0a122841f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a124c4cd33f0a122841f39a2190c12a0173b3efa41d28337bceb3a10db13a254 +size 225549 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_b5ddbe7301ffe322a095.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_b5ddbe7301ffe322a095.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c848f5c5d7feec45f1130231b57f7e384b5882a9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_b5ddbe7301ffe322a095.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5ddbe7301ffe322a09552d3fdd124fee5d35a61b26f26249f8e47f234d90f88 +size 271015 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_e36b7bc1deaaf550e6d8.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_e36b7bc1deaaf550e6d8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a840b7c512ae74a7c1aa1458fc3ced852ed7e0d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_500_e36b7bc1deaaf550e6d8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e36b7bc1deaaf550e6d81ebc35f96ae4b7e67097c6a4890fe65d5da7dae09dc6 +size 192177 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_02c1aae0e691f96ea5ad.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_02c1aae0e691f96ea5ad.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b97a3fc1de797ca0bd839f6a31691fb661a0fd6c --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_02c1aae0e691f96ea5ad.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02c1aae0e691f96ea5ad5306c71f17c454dfc1623713fe1753165d72a108a174 +size 236276 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_2dc2540d510c8648dc00.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_2dc2540d510c8648dc00.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9acaa2af3beb8990022cc12d699b00184ec377c --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_2dc2540d510c8648dc00.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dc2540d510c8648dc000abc1321c2d78ecb3fb3d2a9c46bb012554c2fdb2447 +size 247598 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_3af3ff96013a3f3a3e81.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_3af3ff96013a3f3a3e81.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba82b1357d55be4493a8fce2ae7253bb7a4eef47 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_3af3ff96013a3f3a3e81.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af3ff96013a3f3a3e810246a9f2ac50eafa426ffb9e9842c5584db69acdd52a +size 235006 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_86f8df0cf9c178a9bc20.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_86f8df0cf9c178a9bc20.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87320bdf112d9fc76319c7e132a541c40ded7e2f --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_86f8df0cf9c178a9bc20.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86f8df0cf9c178a9bc205c736298720c84f30c56ccf5c1bc42e7a3c8fb8a22b9 +size 208037 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_8ae4464b03faf92d8fcc.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_8ae4464b03faf92d8fcc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..90213e776b8c06e836df9c261d08d63e0759ee9a --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_8ae4464b03faf92d8fcc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ae4464b03faf92d8fcc6b1be690fc26fd2fc1be9cd0c51f1a6d6b41c12f0176 +size 200192 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_90413783c59d55141104.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_90413783c59d55141104.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e4267c98f4e0ffd2e02b5818e4362c6b9d902cb7 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_90413783c59d55141104.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90413783c59d55141104700f624ed9a70a38de17983d65c487cb0a3c2862d7c4 +size 291439 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_9db48cbc78876e078e80.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_9db48cbc78876e078e80.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a5a77e03aec913f9e0be4b06fa97bfa61818f444 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_9db48cbc78876e078e80.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9db48cbc78876e078e80e5ebdd7dfbd361e2bba79bb1b9c9e0a93fc4c58ffa51 +size 288888 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_c2ea37b78886ecef23af.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_c2ea37b78886ecef23af.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18b3d9482a07360e0e5f33fe9259984d2b5fb9b9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_c2ea37b78886ecef23af.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2ea37b78886ecef23af2a167f3009211e1399bcddc93094b457c746162fcb10 +size 182572 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_ea52c20a941a74fc52e8.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_ea52c20a941a74fc52e8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cb3b6107de570d84fc4f50f108f8570c33436483 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_ea52c20a941a74fc52e8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea52c20a941a74fc52e88ca271d58442d20dbfaf05582f041a367bc867edb3db +size 193724 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_fc14a16bac32611ba42f.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_fc14a16bac32611ba42f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..165b2f05987cc5bd33bf5f8a829fb1c0b7b1ec12 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/generated_600_fc14a16bac32611ba42f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc14a16bac32611ba42f99ca44e4dcffd0c53658bfdb7ddfc3700f2612c5418c +size 209477 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/output.log b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c267bcb7cba96c65c492a43cb9064976179c7f08 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/output.log @@ -0,0 +1,10736 @@ +[May 6, 2026 - 19:10:18 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 6, 2026 - 19:10:18 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 6, 2026 - 19:10:18 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 6, 2026 - 19:10:18 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 6, 2026 - 19:10:18 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 6, 2026 - 19:10:18 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 6, 2026 - 19:10:18 | INFO | fastgen.trainer:run:77 ] Starting training +[May 6, 2026 - 19:10:18 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 6, 2026 - 19:10:18 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 6, 2026 - 19:10:45 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 6, 2026 - 19:10:57 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 6, 2026 - 19:10:57 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 6, 2026 - 19:10:57 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 6, 2026 - 19:10:57 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 6, 2026 - 19:11:41 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 6, 2026 - 19:11:41 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 6, 2026 - 19:11:41 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 6, 2026 - 19:11:41 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 6, 2026 - 19:11:41 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 6, 2026 - 19:11:41 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 6, 2026 - 19:11:43 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 6, 2026 - 19:11:43 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 6, 2026 - 19:11:43 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 6, 2026 - 19:11:43 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 6, 2026 - 19:11:44 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 6, 2026 - 19:11:45 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.9s +[May 6, 2026 - 19:11:45 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 6, 2026 - 19:11:45 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 6, 2026 - 19:11:57 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 6, 2026 - 19:11:57 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.1s +[May 6, 2026 - 19:11:57 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 6, 2026 - 19:11:57 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 6, 2026 - 19:11:57 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 6, 2026 - 19:11:57 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[May 6, 2026 - 19:11:57 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17136.85 M total params (logical). +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4285.56 M total params LOCAL on rank 0. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 1. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 2. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 3. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 6, 2026 - 19:12:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 6, 2026 - 19:12:06 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.394658 37.517811 37.329056 +peak_gpu_mem_gb 57.011160 57.011160 57.011160 +peak_gpu_mem_reserved_gb 59.156250 59.156250 59.156250 +util 79.250000 84.000000 74.000000 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.49 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 10.63 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 10.11 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.79 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:12:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.570598 37.694229 37.504654 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 94.750000 99.000000 87.000000 +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 45.96 seconds +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.05 +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 10.15 +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 1.58 +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:13:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.571432 37.694622 37.504803 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 93.250000 99.000000 88.000000 +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 49.03 seconds +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 13.55 +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 10.17 +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 1.58 +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:14:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.571605 37.694016 37.505131 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.211426 61.212891 61.210938 +util 93.000000 97.000000 89.000000 +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 43.48 seconds +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 10.80 +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 10.17 +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 1.57 +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:15:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.593053 37.714157 37.526787 +peak_gpu_mem_gb 96.121632 96.123194 96.121111 +peak_gpu_mem_reserved_gb 102.568848 102.658203 102.539062 +util 57.750000 65.000000 51.000000 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 161.95 seconds +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 72.09 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 8.45 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.44 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8301 iter count: 1.0 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8301 iter count: 1.0 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8301 iter count: 1.0 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8301iter count: 1.0 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:17:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.593265 37.714214 37.527077 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.758789 64.820312 64.738281 +util 95.500000 99.000000 91.000000 +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 49.90 seconds +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 14.02 +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 10.14 +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 1.58 +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:18:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.593714 37.714214 37.528233 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656250 64.703125 64.640625 +util 92.250000 99.000000 87.000000 +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 48.92 seconds +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 13.48 +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 10.15 +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 1.57 +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:19:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.593837 37.714371 37.528358 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.631836 64.664062 64.621094 +util 96.000000 100.000000 89.000000 +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 48.75 seconds +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 13.45 +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 10.13 +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 1.57 +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 8-------------------- +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:20:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.594012 37.714394 37.528477 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.714844 64.761719 64.699219 +util 94.750000 99.000000 89.000000 +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 43.64 seconds +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 10.90 +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 10.14 +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 1.58 +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 9-------------------- +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:21:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.594966 37.715416 37.529057 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.728027 105.882812 105.263672 +util 66.250000 74.000000 59.000000 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 164.02 seconds +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 73.36 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 8.41 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.39 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 10-------------------- +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8125 iter count: 1.0 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8125 iter count: 1.0 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8125 iter count: 1.0 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8125iter count: 1.0 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:23:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.595325 37.715771 37.529140 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.680664 64.859375 64.621094 +util 92.750000 99.000000 89.000000 +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 47.40 seconds +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 12.76 +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 10.15 +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 1.59 +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 11-------------------- +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:24:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.595985 37.716721 37.529560 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.651367 64.742188 64.621094 +util 95.000000 99.000000 90.000000 +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 48.89 seconds +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 13.48 +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 10.14 +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 1.58 +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 12-------------------- +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0060 iter count: 1.0 +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:25:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596575 37.718639 37.529697 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.661133 64.781250 64.621094 +util 94.500000 99.000000 90.000000 +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 48.75 seconds +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 13.44 +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 10.14 +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 1.57 +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 13-------------------- +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:26:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.596731 37.718700 37.529953 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.622070 64.742188 64.582031 +util 96.500000 100.000000 89.000000 +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 46.12 seconds +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 12.14 +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 10.13 +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 1.57 +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 14-------------------- +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:27:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597398 37.718750 37.531975 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.186035 105.263672 105.160156 +util 65.000000 70.000000 60.000000 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 158.75 seconds +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 70.78 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 8.39 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.40 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 15-------------------- +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7666 iter count: 1.0 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7666 iter count: 1.0 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7666 iter count: 1.0 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7666iter count: 1.0 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:29:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597546 37.718895 37.532169 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 64.976562 64.875000 +util 97.500000 100.000000 93.000000 +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 44.83 seconds +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 11.48 +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 10.14 +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 1.58 +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 16-------------------- +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:30:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597705 37.719372 37.532219 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871094 64.917969 64.855469 +util 95.750000 99.000000 89.000000 +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 46.23 seconds +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 12.19 +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 10.14 +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 1.57 +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 17-------------------- +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0060 iter count: 1.0 +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:31:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597980 37.719383 37.532322 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749023 64.898438 64.699219 +util 93.250000 99.000000 87.000000 +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 46.02 seconds +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 12.08 +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 10.15 +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 1.58 +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 18-------------------- +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:31:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598055 37.719383 37.532368 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.851562 64.898438 64.835938 +util 97.000000 99.000000 93.000000 +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 46.14 seconds +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 12.15 +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 10.14 +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 1.56 +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 19-------------------- +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:32:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.597985 37.718342 37.532574 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.195801 105.302734 105.160156 +util 64.750000 70.000000 57.000000 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 163.62 seconds +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 73.22 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 8.41 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.39 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 20-------------------- +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8955 iter count: 1.0 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8955 iter count: 1.0 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8955 iter count: 1.0 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8955iter count: 1.0 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:35:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598042 37.718460 37.532627 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.807617 64.898438 64.777344 +util 95.500000 99.000000 88.000000 +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 44.61 seconds +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 11.38 +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 10.13 +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 1.57 +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 21-------------------- +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:36:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598171 37.718716 37.532776 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.675781 64.757812 64.429688 +util 95.250000 99.000000 88.000000 +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 45.99 seconds +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 12.07 +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 10.14 +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 1.57 +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 22-------------------- +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0060 iter count: 1.0 +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:36:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598328 37.719276 37.532864 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.617188 64.679688 64.429688 +util 97.500000 99.000000 94.000000 +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 45.99 seconds +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 12.04 +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 10.14 +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 1.59 +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 23-------------------- +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:37:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598452 37.719517 37.532974 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.602539 64.660156 64.429688 +util 94.500000 99.000000 87.000000 +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 46.20 seconds +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 12.17 +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 10.14 +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 1.56 +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 24-------------------- +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:38:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598330 37.718830 37.533031 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.093262 105.126953 105.082031 +util 67.250000 75.000000 60.000000 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 158.50 seconds +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 70.63 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 8.41 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.40 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 25-------------------- +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8115 iter count: 1.0 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8115 iter count: 1.0 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8115 iter count: 1.0 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8115iter count: 1.0 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0056 iter count: 1.0 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:41:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598380 37.718887 37.533089 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.793457 64.841797 64.777344 +util 95.000000 99.000000 89.000000 +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 44.73 seconds +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 11.42 +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 10.13 +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 1.60 +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 26-------------------- +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:41:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598523 37.719341 37.533203 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.676270 64.724609 64.660156 +util 93.750000 100.000000 87.000000 +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 48.82 seconds +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 13.48 +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 10.15 +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 1.56 +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 27-------------------- +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0062 iter count: 1.0 +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:42:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598788 37.720371 37.533218 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 65.833496 66.222656 64.666016 +util 95.500000 99.000000 90.000000 +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 48.64 seconds +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 13.39 +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 10.14 +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 1.57 +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 28-------------------- +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:43:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598807 37.720356 37.533298 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656738 64.705078 64.640625 +util 96.500000 99.000000 92.000000 +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 46.09 seconds +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 12.10 +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 10.14 +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 1.57 +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 29-------------------- +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:44:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598579 37.719051 37.533451 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.156738 105.160156 105.146484 +util 64.250000 72.000000 57.000000 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 165.89 seconds +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 74.34 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 8.43 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.39 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 30-------------------- +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8584 iter count: 1.0 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8584 iter count: 1.0 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8584 iter count: 1.0 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8584iter count: 1.0 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:47:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598718 37.719349 37.533531 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.836914 64.839844 64.835938 +util 95.000000 99.000000 89.000000 +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 44.81 seconds +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 11.44 +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 10.12 +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 1.57 +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 31-------------------- +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:47:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.598794 37.719612 37.533554 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700195 64.820312 64.660156 +util 95.250000 99.000000 88.000000 +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 45.94 seconds +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 12.03 +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 10.15 +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 1.57 +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 32-------------------- +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:48:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599126 37.720860 37.533627 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.778320 64.796875 64.722656 +util 93.250000 99.000000 88.000000 +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 43.42 seconds +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 10.79 +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 10.12 +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 1.56 +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 33-------------------- +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0058 iter count: 1.0 +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:49:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599151 37.720860 37.533627 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.792969 64.839844 64.777344 +util 93.500000 100.000000 88.000000 +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 48.66 seconds +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 13.38 +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 10.14 +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 1.56 +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 34-------------------- +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:50:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599186 37.720875 37.533730 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.156738 105.205078 105.140625 +util 66.250000 71.000000 60.000000 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 160.68 seconds +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 71.74 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 8.44 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.40 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 35-------------------- +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7764 iter count: 1.0 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7764 iter count: 1.0 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7764 iter count: 1.0 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7764iter count: 1.0 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:52:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599049 37.720245 37.533726 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.822266 64.898438 64.796875 +util 95.250000 99.000000 90.000000 +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 47.09 seconds +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 12.62 +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 10.15 +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 1.56 +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 36-------------------- +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599049 37.720280 37.533714 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.783203 64.859375 64.757812 +util 93.750000 99.000000 87.000000 +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 43.57 seconds +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 10.86 +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 10.14 +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 1.57 +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 37-------------------- +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:54:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599065 37.720268 37.533726 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.792969 64.839844 64.777344 +util 95.000000 99.000000 89.000000 +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 48.62 seconds +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 13.39 +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 10.13 +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 1.58 +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 38-------------------- +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:55:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599070 37.720276 37.533726 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.820312 64.816406 +util 95.500000 99.000000 90.000000 +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 46.01 seconds +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 12.09 +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 10.14 +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 1.56 +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 39-------------------- +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:55:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599108 37.720287 37.533806 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.215332 105.263672 105.199219 +util 66.250000 70.000000 62.000000 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 160.50 seconds +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 71.65 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 8.38 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.39 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 40-------------------- +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7822 iter count: 1.0 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7822 iter count: 1.0 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7822 iter count: 1.0 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7822iter count: 1.0 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:58:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599098 37.720287 37.533810 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.886230 64.894531 64.861328 +util 92.750000 98.000000 88.000000 +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 49.76 seconds +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 13.96 +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 10.12 +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 1.58 +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 41-------------------- +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 19:59:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599125 37.720314 37.533813 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.861816 64.875000 64.822266 +util 96.250000 100.000000 92.000000 +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 46.03 seconds +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 12.08 +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 10.14 +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 1.58 +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 42-------------------- +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:00:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599171 37.720322 37.533936 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.876465 64.894531 64.822266 +util 95.000000 99.000000 90.000000 +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 43.38 seconds +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 10.78 +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 10.13 +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 1.56 +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 43-------------------- +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:00:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599291 37.720825 37.533920 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.856934 64.875000 64.802734 +util 94.750000 99.000000 90.000000 +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 45.97 seconds +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 12.07 +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 10.13 +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 1.57 +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 44-------------------- +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:01:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599141 37.720169 37.533955 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.166504 105.185547 105.160156 +util 65.000000 71.000000 57.000000 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 160.89 seconds +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 71.84 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 8.45 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.39 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 45-------------------- +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7725 iter count: 1.0 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7725 iter count: 1.0 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7725 iter count: 1.0 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7725iter count: 1.0 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:04:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599183 37.720169 37.534096 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871582 64.978516 64.835938 +util 95.250000 99.000000 88.000000 +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 49.80 seconds +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 13.98 +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 10.13 +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 1.57 +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 46-------------------- +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:05:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599183 37.720135 37.534130 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.720215 64.796875 64.490234 +util 95.250000 99.000000 90.000000 +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 46.06 seconds +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 12.10 +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 10.13 +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 1.58 +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 47-------------------- +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:05:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599195 37.720150 37.534142 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.642090 64.699219 64.470703 +util 95.250000 99.000000 91.000000 +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 48.68 seconds +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 13.39 +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 10.16 +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 1.57 +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 48-------------------- +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:06:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599343 37.720619 37.534264 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.715332 64.796875 64.470703 +util 93.500000 100.000000 88.000000 +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 48.93 seconds +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 13.51 +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 10.15 +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 1.57 +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 49-------------------- +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:07:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599194 37.720013 37.534241 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.410645 105.927734 105.238281 +util 66.000000 73.000000 61.000000 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 158.43 seconds +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 70.59 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 8.44 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.39 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 50-------------------- +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7881 iter count: 1.0 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7881 iter count: 1.0 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7881 iter count: 1.0 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7881iter count: 1.0 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:10:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599212 37.720013 37.534264 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.837402 64.841797 64.835938 +util 93.500000 99.000000 88.000000 +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 47.37 seconds +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 12.76 +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 10.14 +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 1.56 +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 51-------------------- +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:10:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599210 37.720013 37.534248 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734863 64.783203 64.718750 +util 96.000000 99.000000 92.000000 +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 46.08 seconds +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 12.08 +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 10.20 +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 1.56 +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 52-------------------- +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:11:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599285 37.720013 37.534443 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.847168 64.880859 64.835938 +util 94.500000 99.000000 90.000000 +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 46.17 seconds +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 12.15 +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 10.17 +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 1.58 +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 53-------------------- +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:12:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599638 37.721241 37.534420 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.793457 64.796875 64.783203 +util 95.250000 100.000000 89.000000 +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 45.92 seconds +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 12.03 +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 10.17 +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 1.57 +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 54-------------------- +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:13:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599468 37.720516 37.534435 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.142090 105.146484 105.140625 +util 62.250000 69.000000 55.000000 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 160.77 seconds +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 71.79 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 8.39 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.39 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 55-------------------- +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7568 iter count: 1.0 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7568 iter count: 1.0 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7568 iter count: 1.0 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7568iter count: 1.0 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:15:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599486 37.720528 37.534489 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.885742 64.914062 64.800781 +util 92.750000 98.000000 88.000000 +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 44.49 seconds +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 11.30 +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 10.15 +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 1.57 +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 56-------------------- +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:16:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599493 37.720531 37.534489 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.832031 64.894531 64.644531 +util 95.500000 100.000000 90.000000 +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 48.55 seconds +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 13.36 +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 10.11 +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 1.57 +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 57-------------------- +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:17:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599504 37.720535 37.534508 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.524414 64.703125 64.464844 +util 96.000000 99.000000 92.000000 +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 45.90 seconds +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 12.02 +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 10.12 +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 1.58 +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 58-------------------- +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:18:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599523 37.720528 37.534489 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.885742 64.953125 64.683594 +util 96.750000 100.000000 90.000000 +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 45.86 seconds +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 12.00 +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 10.14 +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 1.56 +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 59-------------------- +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:19:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599469 37.720291 37.534492 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.156738 105.160156 105.146484 +util 63.250000 70.000000 56.000000 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 160.62 seconds +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 71.71 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 8.40 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.39 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 60-------------------- +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7646 iter count: 1.0 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7646 iter count: 1.0 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7646 iter count: 1.0 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7646iter count: 1.0 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599493 37.720291 37.534584 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 64.914062 64.859375 +util 97.000000 100.000000 93.000000 +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 49.95 seconds +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 14.03 +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 10.14 +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 1.59 +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 61-------------------- +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:22:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599531 37.720341 37.534584 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871094 64.933594 64.683594 +util 94.500000 99.000000 90.000000 +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 45.96 seconds +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 12.06 +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 10.14 +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 1.56 +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 62-------------------- +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:23:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599609 37.720600 37.534584 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.905273 64.933594 64.820312 +util 95.750000 100.000000 90.000000 +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 43.33 seconds +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 10.75 +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 10.12 +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 1.57 +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 63-------------------- +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:23:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599607 37.720604 37.534584 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.866211 64.894531 64.781250 +util 94.250000 99.000000 89.000000 +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 46.03 seconds +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 12.09 +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 10.14 +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 1.56 +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 64-------------------- +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:24:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599410 37.719688 37.534695 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.210449 105.218750 105.185547 +util 67.250000 74.000000 60.000000 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 163.39 seconds +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 73.09 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 8.42 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.39 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 65-------------------- +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7314 iter count: 1.0 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7314 iter count: 1.0 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7314 iter count: 1.0 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7314iter count: 1.0 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:27:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599420 37.719677 37.534721 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.631836 64.839844 64.562500 +util 95.500000 99.000000 90.000000 +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 46.99 seconds +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 12.56 +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 10.16 +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 1.57 +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 66-------------------- +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599559 37.720207 37.534725 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.548828 64.683594 64.503906 +util 97.000000 100.000000 93.000000 +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 46.12 seconds +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 12.13 +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 10.14 +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 1.58 +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 67-------------------- +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599559 37.720165 37.534706 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.583008 64.761719 64.523438 +util 94.000000 100.000000 87.000000 +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 48.65 seconds +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 13.40 +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 10.13 +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 1.57 +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 68-------------------- +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:29:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599622 37.720417 37.534706 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.563477 64.742188 64.503906 +util 95.500000 99.000000 90.000000 +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 48.63 seconds +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 13.40 +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 10.14 +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 1.56 +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 69-------------------- +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:30:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599524 37.719730 37.534775 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.161621 105.341797 105.101562 +util 66.500000 74.000000 61.000000 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 160.66 seconds +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 71.75 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 8.40 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.39 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 70-------------------- +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7363 iter count: 1.0 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7363 iter count: 1.0 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7363 iter count: 1.0 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7363iter count: 1.0 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:33:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599524 37.719730 37.534794 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.832520 64.855469 64.763672 +util 93.500000 99.000000 86.000000 +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 47.11 seconds +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 12.63 +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 10.15 +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 1.57 +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 71-------------------- +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:34:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599710 37.720360 37.534798 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.744629 64.757812 64.705078 +util 95.750000 99.000000 89.000000 +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 45.98 seconds +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 12.03 +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 10.16 +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 1.59 +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 72-------------------- +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:34:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599909 37.721107 37.534832 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.729980 64.757812 64.646484 +util 95.000000 99.000000 88.000000 +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 48.59 seconds +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 13.36 +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 10.17 +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 1.58 +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 73-------------------- +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:35:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600045 37.721600 37.534866 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749512 64.777344 64.666016 +util 96.250000 99.000000 90.000000 +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 48.82 seconds +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 13.48 +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 10.14 +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 1.57 +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 74-------------------- +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:36:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599654 37.719894 37.534866 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.127441 105.205078 105.101562 +util 65.250000 70.000000 58.000000 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 160.20 seconds +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 71.50 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 8.39 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.40 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 75-------------------- +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6992 iter count: 1.0 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6992 iter count: 1.0 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6992 iter count: 1.0 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6992iter count: 1.0 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:39:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599660 37.719894 37.534843 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871582 64.919922 64.855469 +util 94.250000 100.000000 89.000000 +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 49.88 seconds +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 14.02 +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 10.14 +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 1.57 +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 76-------------------- +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:40:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599863 37.720634 37.534916 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.808105 64.841797 64.796875 +util 94.750000 99.000000 90.000000 +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 48.61 seconds +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 13.38 +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 10.13 +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 1.57 +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 77-------------------- +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:40:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600056 37.721386 37.534931 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.725098 64.802734 64.699219 +util 93.750000 99.000000 88.000000 +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 48.58 seconds +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 13.36 +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 10.15 +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 1.58 +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 78-------------------- +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:41:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600064 37.721390 37.534939 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.729980 64.822266 64.699219 +util 93.750000 99.000000 87.000000 +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 48.55 seconds +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 13.35 +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 10.14 +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 1.58 +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 79-------------------- +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:42:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599691 37.719784 37.534946 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.146973 105.166016 105.140625 +util 64.750000 73.000000 58.000000 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 155.47 seconds +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 69.15 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 8.38 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.39 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 80-------------------- +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7412 iter count: 1.0 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7412 iter count: 1.0 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7412 iter count: 1.0 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7412iter count: 1.0 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:45:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599694 37.719772 37.534946 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.808105 64.816406 64.783203 +util 93.250000 99.000000 88.000000 +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 44.37 seconds +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 11.26 +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 10.13 +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 1.58 +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 81-------------------- +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:45:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.599889 37.720520 37.534946 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.710449 64.738281 64.626953 +util 95.250000 99.000000 91.000000 +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 43.47 seconds +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 10.78 +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 10.16 +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 1.58 +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 82-------------------- +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:46:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600332 37.722290 37.534946 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700684 64.718750 64.646484 +util 94.250000 99.000000 90.000000 +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 45.98 seconds +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 12.08 +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 10.11 +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 1.55 +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 83-------------------- +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:47:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600361 37.722267 37.534996 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.769043 64.796875 64.685547 +util 94.750000 99.000000 90.000000 +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 45.97 seconds +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 12.06 +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 10.13 +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 1.57 +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 84-------------------- +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:48:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600309 37.721802 37.535114 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.229980 105.238281 105.205078 +util 62.500000 69.000000 55.000000 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 160.33 seconds +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 71.55 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 8.41 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.40 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 85-------------------- +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7109 iter count: 1.0 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7109 iter count: 1.0 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7109 iter count: 1.0 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7109iter count: 1.0 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:50:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600181 37.721283 37.535133 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.837402 64.841797 64.835938 +util 97.000000 100.000000 93.000000 +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 46.95 seconds +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 12.56 +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 10.14 +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 1.56 +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 86-------------------- +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:51:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600198 37.721298 37.535141 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.759277 64.796875 64.646484 +util 93.500000 99.000000 89.000000 +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 45.94 seconds +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 12.04 +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 10.14 +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 1.58 +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 87-------------------- +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:52:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600195 37.721306 37.535141 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.778809 64.796875 64.724609 +util 95.750000 100.000000 91.000000 +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 46.03 seconds +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 12.10 +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 10.14 +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 1.56 +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 88-------------------- +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:53:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600196 37.721333 37.535141 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.783691 64.816406 64.685547 +util 95.000000 100.000000 88.000000 +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 46.04 seconds +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 12.11 +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 10.14 +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 1.57 +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 89-------------------- +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:53:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600193 37.721310 37.535141 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.308105 105.810547 105.140625 +util 63.250000 69.000000 59.000000 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 163.34 seconds +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 73.08 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 8.41 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.39 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 90-------------------- +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7314 iter count: 1.0 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7314 iter count: 1.0 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7314 iter count: 1.0 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7314iter count: 1.0 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:56:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600180 37.721146 37.535156 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.783691 64.835938 64.626953 +util 96.750000 100.000000 93.000000 +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 44.44 seconds +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 11.31 +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 10.12 +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 1.56 +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 91-------------------- +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:57:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600196 37.721191 37.535168 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.710449 64.757812 64.568359 +util 96.500000 100.000000 90.000000 +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 48.45 seconds +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 13.31 +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 10.12 +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 1.58 +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 92-------------------- +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:58:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600197 37.721161 37.535168 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734863 64.777344 64.607422 +util 93.500000 99.000000 89.000000 +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 45.91 seconds +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 12.03 +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 10.16 +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 1.58 +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 93-------------------- +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:58:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600204 37.721191 37.535168 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749512 64.796875 64.607422 +util 94.500000 99.000000 88.000000 +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 43.26 seconds +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 10.72 +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 10.12 +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 1.57 +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 94-------------------- +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 20:59:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600212 37.721180 37.535175 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.259277 105.439453 105.199219 +util 63.500000 69.000000 56.000000 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 157.81 seconds +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 70.30 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 8.41 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.40 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 95-------------------- +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7744 iter count: 1.0 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7744 iter count: 1.0 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7744 iter count: 1.0 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7744iter count: 1.0 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:02:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600252 37.721191 37.535175 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.690918 64.777344 64.431641 +util 96.250000 99.000000 92.000000 +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 44.77 seconds +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 11.45 +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 10.15 +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 1.57 +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 96-------------------- +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:02:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600247 37.721184 37.535175 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.593262 64.640625 64.451172 +util 93.500000 99.000000 88.000000 +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 45.92 seconds +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.03 +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 10.15 +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 1.58 +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 97-------------------- +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:03:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600247 37.721184 37.535175 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.622559 64.679688 64.451172 +util 96.500000 100.000000 92.000000 +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 46.08 seconds +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.13 +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 10.14 +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 1.55 +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 98-------------------- +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:04:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600454 37.721939 37.535175 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749512 64.958984 64.679688 +util 96.000000 100.000000 90.000000 +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 46.11 seconds +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 12.11 +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 10.14 +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 1.58 +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 99-------------------- +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:05:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600327 37.721348 37.535229 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.137207 105.185547 105.121094 +util 66.000000 71.000000 62.000000 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 166.11 seconds +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 74.45 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 8.43 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.41 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 100-------------------- +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7627 iter count: 1.0 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7627 iter count: 1.0 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7627 iter count: 1.0 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7627iter count: 1.0 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 21:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 6, 2026 - 21:08:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 17.74 seconds +[May 6, 2026 - 21:08:17 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[May 6, 2026 - 21:08:17 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000100 +[May 6, 2026 - 21:08:37 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 100 +[May 6, 2026 - 21:08:38 | INFO | fastgen.trainer:validate:358 ] Validation iteration 100 +[May 6, 2026 - 21:08:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:08:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 6, 2026 - 21:08:43 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 6, 2026 - 21:08:44 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.475 +[May 6, 2026 - 21:08:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:08:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:08:49 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.967 +[May 6, 2026 - 21:08:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:08:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:08:54 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.911 +[May 6, 2026 - 21:08:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:08:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:08:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.934 +[May 6, 2026 - 21:08:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:09:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:09:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.799 +[May 6, 2026 - 21:09:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:09:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:09:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.681 +[May 6, 2026 - 21:09:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:09:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:09:14 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.012 +[May 6, 2026 - 21:09:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:09:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:09:19 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.174 +[May 6, 2026 - 21:09:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:09:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:09:24 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.356 +[May 6, 2026 - 21:09:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 21:09:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 21:09:29 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.471 +[May 6, 2026 - 21:09:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 100-------------------- +[May 6, 2026 - 21:09:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 6, 2026 - 21:09:46 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 100 +[May 6, 2026 - 21:09:46 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.278 (n=10) +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439661 41.587505 41.365479 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.290039 68.515625 68.214844 +util 94.250000 99.000000 89.000000 +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 101 : avg iteration time 149.92 seconds +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 101 : data loading time 0.00 +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 101 : avg forward pass time 10.50 +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 101 : backward pass time 10.14 +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 101 : optimizer step time 1.58 +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 101-------------------- +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:10:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439507 41.586792 41.365479 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.285156 68.613281 68.175781 +util 94.000000 99.000000 89.000000 +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 102 : avg iteration time 48.84 seconds +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 102 : data loading time 0.00 +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 102 : avg forward pass time 13.43 +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 102 : backward pass time 10.20 +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 102 : optimizer step time 1.59 +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 102-------------------- +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:11:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439511 41.586792 41.365479 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.260742 68.515625 68.175781 +util 95.750000 99.000000 92.000000 +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 103 : avg iteration time 43.43 seconds +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 103 : data loading time 0.00 +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 103 : avg forward pass time 10.76 +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 103 : backward pass time 10.18 +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 103 : optimizer step time 1.57 +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 103-------------------- +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:12:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439426 41.586475 41.365479 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.265625 68.535156 68.175781 +util 93.750000 99.000000 88.000000 +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 104 : avg iteration time 46.13 seconds +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 104 : data loading time 0.00 +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 104 : avg forward pass time 12.12 +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 104 : backward pass time 10.16 +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 104 : optimizer step time 1.58 +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 104-------------------- +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:12:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439766 41.584156 41.366863 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.746094 107.544922 +util 66.250000 74.000000 60.000000 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 105 : avg iteration time 160.64 seconds +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 105 : data loading time 0.00 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 105 : avg forward pass time 71.69 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 105 : backward pass time 8.44 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 105 : optimizer step time 0.40 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 105-------------------- +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7539 iter count: 1.0 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7539 iter count: 1.0 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7539 iter count: 1.0 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7539iter count: 1.0 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:15:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439781 41.584164 41.366898 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.300781 69.894531 67.519531 +util 96.250000 100.000000 90.000000 +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 106 : avg iteration time 44.36 seconds +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 106 : data loading time 0.00 +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 106 : avg forward pass time 11.27 +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 106 : backward pass time 10.12 +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 106 : optimizer step time 1.58 +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 106-------------------- +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:16:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439794 41.584167 41.366882 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.369141 69.992188 67.500000 +util 95.250000 100.000000 89.000000 +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 107 : avg iteration time 48.78 seconds +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 107 : data loading time 0.00 +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 107 : avg forward pass time 13.48 +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 107 : backward pass time 10.14 +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 107 : optimizer step time 1.57 +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 107-------------------- +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:17:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439802 41.584179 41.366852 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.427734 70.070312 67.500000 +util 95.750000 99.000000 90.000000 +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 108 : avg iteration time 48.75 seconds +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 108 : data loading time 0.00 +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 108 : avg forward pass time 13.45 +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 108 : backward pass time 10.14 +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 108 : optimizer step time 1.58 +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 108-------------------- +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:17:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439987 41.584904 41.366859 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.403320 70.031250 67.519531 +util 94.250000 99.000000 88.000000 +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 109 : avg iteration time 46.03 seconds +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 109 : data loading time 0.00 +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 109 : avg forward pass time 12.08 +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 109 : backward pass time 10.14 +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 109 : optimizer step time 1.58 +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 109-------------------- +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:18:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439856 41.584377 41.366871 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.707031 107.564453 +util 63.000000 69.000000 59.000000 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 110 : avg iteration time 160.90 seconds +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 110 : data loading time 0.00 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 110 : avg forward pass time 71.86 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 110 : backward pass time 8.41 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 110 : optimizer step time 0.40 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 110-------------------- +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7324 iter count: 1.0 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7324 iter count: 1.0 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7324 iter count: 1.0 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7324iter count: 1.0 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:21:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439862 41.584381 41.366871 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.251465 67.871094 67.044922 +util 93.250000 99.000000 86.000000 +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 111 : avg iteration time 47.09 seconds +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 111 : data loading time 0.00 +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 111 : avg forward pass time 12.61 +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 111 : backward pass time 10.14 +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 111 : optimizer step time 1.58 +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 111-------------------- +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:22:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439889 41.584381 41.366882 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.832031 67.318359 +util 93.750000 99.000000 88.000000 +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 112 : avg iteration time 45.95 seconds +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 112 : data loading time 0.00 +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 112 : avg forward pass time 12.04 +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 112 : backward pass time 10.15 +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 112 : optimizer step time 1.57 +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 112-------------------- +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:22:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439898 41.584393 41.366886 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.471191 67.929688 67.318359 +util 94.000000 99.000000 87.000000 +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 113 : avg iteration time 43.40 seconds +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 113 : data loading time 0.00 +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 113 : avg forward pass time 10.77 +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 113 : backward pass time 10.13 +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 113 : optimizer step time 1.58 +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 113-------------------- +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:23:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439901 41.584404 41.366913 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.417480 67.714844 67.318359 +util 94.000000 99.000000 90.000000 +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 114 : avg iteration time 43.35 seconds +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 114 : data loading time 0.00 +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 114 : avg forward pass time 10.75 +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 114 : backward pass time 10.13 +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 114 : optimizer step time 1.57 +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 114-------------------- +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:24:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439841 41.584141 41.366901 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.541504 107.648438 107.505859 +util 64.750000 71.000000 57.000000 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 115 : avg iteration time 163.32 seconds +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 115 : data loading time 0.00 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 115 : avg forward pass time 73.05 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 115 : backward pass time 8.42 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 115 : optimizer step time 0.39 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 115-------------------- +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7129 iter count: 1.0 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7129 iter count: 1.0 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7129 iter count: 1.0 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7129iter count: 1.0 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:27:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439839 41.584141 41.366917 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.157227 67.500000 67.042969 +util 95.000000 99.000000 91.000000 +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 116 : avg iteration time 49.71 seconds +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 116 : data loading time 0.00 +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 116 : avg forward pass time 13.94 +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 116 : backward pass time 10.12 +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 116 : optimizer step time 1.59 +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 116-------------------- +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:27:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439852 41.584141 41.366936 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.890625 67.316406 +util 93.500000 99.000000 89.000000 +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 117 : avg iteration time 43.39 seconds +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 117 : data loading time 0.00 +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 117 : avg forward pass time 10.75 +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 117 : backward pass time 10.13 +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 117 : optimizer step time 1.59 +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 117-------------------- +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:28:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439923 41.584389 41.366924 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.250000 67.871094 67.042969 +util 92.750000 99.000000 88.000000 +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 118 : avg iteration time 43.37 seconds +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 118 : data loading time 0.00 +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 118 : avg forward pass time 10.75 +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 118 : backward pass time 10.13 +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 118 : optimizer step time 1.57 +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 118-------------------- +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:29:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439927 41.584415 41.366928 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.519531 67.375000 +util 93.500000 99.000000 89.000000 +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 119 : avg iteration time 46.01 seconds +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 119 : data loading time 0.00 +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 119 : avg forward pass time 12.08 +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 119 : backward pass time 10.14 +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 119 : optimizer step time 1.57 +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 119-------------------- +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:30:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439684 41.583187 41.367134 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.628906 107.623047 +util 66.750000 73.000000 59.000000 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 120 : avg iteration time 163.16 seconds +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 120 : data loading time 0.00 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 120 : avg forward pass time 72.97 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 120 : backward pass time 8.43 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 120 : optimizer step time 0.40 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 120-------------------- +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7998 iter count: 1.0 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7998 iter count: 1.0 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7998 iter count: 1.0 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7998iter count: 1.0 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:32:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439694 41.583233 41.367142 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.137695 67.480469 67.023438 +util 95.000000 100.000000 88.000000 +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 121 : avg iteration time 44.56 seconds +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 121 : data loading time 0.00 +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 121 : avg forward pass time 11.36 +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 121 : backward pass time 10.14 +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 121 : optimizer step time 1.57 +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 121-------------------- +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:33:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439779 41.583431 41.367153 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.832031 67.160156 +util 93.750000 99.000000 88.000000 +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 122 : avg iteration time 46.05 seconds +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 122 : data loading time 0.00 +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 122 : avg forward pass time 12.10 +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 122 : backward pass time 10.14 +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 122 : optimizer step time 1.56 +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 122-------------------- +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:34:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439788 41.583477 41.367153 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.245117 67.500000 67.160156 +util 93.500000 99.000000 87.000000 +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 123 : avg iteration time 46.28 seconds +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 123 : data loading time 0.00 +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 123 : avg forward pass time 12.22 +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 123 : backward pass time 10.14 +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 123 : optimizer step time 1.57 +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 123-------------------- +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:35:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439846 41.583683 41.367168 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.245117 67.500000 67.160156 +util 93.750000 99.000000 87.000000 +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 124 : avg iteration time 46.05 seconds +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 124 : data loading time 0.00 +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 124 : avg forward pass time 12.10 +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 124 : backward pass time 10.15 +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 124 : optimizer step time 1.57 +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 124-------------------- +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:35:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440126 41.584751 41.367168 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.642578 107.589844 +util 65.500000 71.000000 59.000000 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 125 : avg iteration time 158.18 seconds +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 125 : data loading time 0.00 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 125 : avg forward pass time 70.50 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 125 : backward pass time 8.41 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 125 : optimizer step time 0.40 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 125-------------------- +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6143 iter count: 1.0 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6143 iter count: 1.0 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6143 iter count: 1.0 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6143iter count: 1.0 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:38:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440145 41.584831 41.367168 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.485840 67.871094 67.357422 +util 94.250000 100.000000 88.000000 +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 126 : avg iteration time 49.68 seconds +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 126 : data loading time 0.00 +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 126 : avg forward pass time 13.91 +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 126 : backward pass time 10.14 +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 126 : optimizer step time 1.58 +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 126-------------------- +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:39:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440184 41.585003 41.367172 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.339355 67.753906 67.201172 +util 96.000000 99.000000 91.000000 +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 127 : avg iteration time 43.38 seconds +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 127 : data loading time 0.00 +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 127 : avg forward pass time 10.76 +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 127 : backward pass time 10.14 +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 127 : optimizer step time 1.58 +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 127-------------------- +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:40:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440373 41.585712 41.367218 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.515137 67.929688 67.376953 +util 95.250000 100.000000 89.000000 +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 128 : avg iteration time 43.37 seconds +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 128 : data loading time 0.00 +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 128 : avg forward pass time 10.78 +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 128 : backward pass time 10.13 +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 128 : optimizer step time 1.57 +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 128-------------------- +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:40:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440391 41.585762 41.367210 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.231934 67.910156 67.005859 +util 96.250000 100.000000 92.000000 +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 129 : avg iteration time 43.32 seconds +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 129 : data loading time 0.00 +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 129 : avg forward pass time 10.74 +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 129 : backward pass time 10.13 +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 129 : optimizer step time 1.58 +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 129-------------------- +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:41:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440035 41.584267 41.367222 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.648438 107.623047 +util 64.000000 69.000000 57.000000 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 130 : avg iteration time 160.63 seconds +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 130 : data loading time 0.00 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 130 : avg forward pass time 71.71 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 130 : backward pass time 8.42 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 130 : optimizer step time 0.40 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 130-------------------- +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7119 iter count: 1.0 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7119 iter count: 1.0 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7119 iter count: 1.0 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7119iter count: 1.0 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:44:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440101 41.584267 41.367313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.153809 67.480469 67.044922 +util 93.500000 99.000000 89.000000 +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 131 : avg iteration time 49.93 seconds +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 131 : data loading time 0.00 +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 131 : avg forward pass time 14.05 +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 131 : backward pass time 10.13 +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 131 : optimizer step time 1.57 +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 131-------------------- +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:44:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440184 41.584602 41.367313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.251465 67.460938 67.181641 +util 49.000000 56.000000 41.000000 +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 132 : avg iteration time 50.07 seconds +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 132 : data loading time 0.00 +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 132 : avg forward pass time 13.52 +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 132 : backward pass time 10.61 +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 132 : optimizer step time 2.27 +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 132-------------------- +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:45:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.09 seconds +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440493 41.585815 41.367313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.222168 67.285156 67.201172 +util 52.750000 72.000000 34.000000 +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 133 : avg iteration time 78.18 seconds +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 133 : data loading time 0.00 +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 133 : avg forward pass time 27.74 +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 133 : backward pass time 10.21 +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 133 : optimizer step time 2.18 +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 133-------------------- +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:47:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.09 seconds +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440210 41.584721 41.367313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.256348 67.421875 67.201172 +util 74.500000 91.000000 60.000000 +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 134 : avg iteration time 87.30 seconds +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 134 : data loading time 0.00 +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 134 : avg forward pass time 32.39 +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 134 : backward pass time 10.25 +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 134 : optimizer step time 2.01 +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 134-------------------- +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:48:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.10 seconds +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440064 41.584084 41.367344 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.526855 107.648438 107.486328 +util 58.000000 64.000000 54.000000 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 135 : avg iteration time 217.47 seconds +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 135 : data loading time 0.01 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 135 : avg forward pass time 92.67 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 135 : backward pass time 13.16 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 135 : optimizer step time 0.40 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 135-------------------- +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7051 iter count: 1.0 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7051 iter count: 1.0 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7051 iter count: 1.0 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7051iter count: 1.0 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:52:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440064 41.584084 41.367336 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.681152 67.767578 67.421875 +util 96.250000 100.000000 91.000000 +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 136 : avg iteration time 47.33 seconds +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 136 : data loading time 0.00 +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 136 : avg forward pass time 12.70 +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 136 : backward pass time 10.18 +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 136 : optimizer step time 1.59 +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 136-------------------- +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:52:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440144 41.584305 41.367382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.603027 67.669922 67.402344 +util 94.750000 99.000000 88.000000 +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 137 : avg iteration time 46.17 seconds +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 137 : data loading time 0.00 +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 137 : avg forward pass time 12.12 +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 137 : backward pass time 10.16 +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 137 : optimizer step time 1.56 +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 137-------------------- +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:53:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440349 41.585232 41.367336 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.568848 67.669922 67.265625 +util 94.250000 100.000000 87.000000 +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 138 : avg iteration time 48.14 seconds +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 138 : data loading time 0.00 +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 138 : avg forward pass time 12.52 +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 138 : backward pass time 10.15 +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 138 : optimizer step time 1.56 +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 138-------------------- +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:54:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440422 41.585434 41.367382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.607910 67.669922 67.421875 +util 94.250000 99.000000 90.000000 +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 139 : avg iteration time 50.63 seconds +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 139 : data loading time 0.00 +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 139 : avg forward pass time 14.34 +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 139 : backward pass time 10.13 +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 139 : optimizer step time 1.64 +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 139-------------------- +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:55:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.02 seconds +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439940 41.583599 41.367336 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.707031 107.544922 +util 64.250000 70.000000 57.000000 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 140 : avg iteration time 165.84 seconds +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 140 : data loading time 0.00 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 140 : avg forward pass time 74.31 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 140 : backward pass time 8.41 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 140 : optimizer step time 0.41 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 140-------------------- +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7051 iter count: 1.0 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7051 iter count: 1.0 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7051 iter count: 1.0 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7051iter count: 1.0 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:58:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.439984 41.583630 41.367386 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.646973 67.929688 67.552734 +util 96.000000 100.000000 90.000000 +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 141 : avg iteration time 47.35 seconds +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 141 : data loading time 0.00 +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 141 : avg forward pass time 12.74 +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 141 : backward pass time 10.15 +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 141 : optimizer step time 1.58 +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 141-------------------- +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:58:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440064 41.584038 41.367363 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.764160 67.812500 67.748047 +util 94.750000 99.000000 88.000000 +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 142 : avg iteration time 46.04 seconds +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 142 : data loading time 0.00 +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 142 : avg forward pass time 12.09 +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 142 : backward pass time 10.13 +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 142 : optimizer step time 1.57 +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 142-------------------- +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 21:59:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440390 41.585335 41.367367 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.500488 67.929688 67.357422 +util 94.500000 99.000000 88.000000 +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 143 : avg iteration time 46.14 seconds +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 143 : data loading time 0.00 +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 143 : avg forward pass time 12.15 +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 143 : backward pass time 10.14 +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 143 : optimizer step time 1.57 +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 143-------------------- +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:00:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440391 41.585335 41.367355 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.686035 67.851562 67.630859 +util 96.000000 100.000000 90.000000 +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 144 : avg iteration time 43.68 seconds +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 144 : data loading time 0.00 +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 144 : avg forward pass time 10.81 +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 144 : backward pass time 10.15 +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 144 : optimizer step time 1.56 +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 144-------------------- +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:01:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440478 41.585567 41.367367 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.726562 107.603516 +util 63.750000 69.000000 57.000000 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 145 : avg iteration time 161.13 seconds +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 145 : data loading time 0.00 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 145 : avg forward pass time 71.93 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 145 : backward pass time 8.43 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 145 : optimizer step time 0.39 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 145-------------------- +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7129 iter count: 1.0 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7129 iter count: 1.0 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7129 iter count: 1.0 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7129iter count: 1.0 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:03:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440302 41.584873 41.367367 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.660156 67.707031 67.519531 +util 93.500000 99.000000 86.000000 +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 146 : avg iteration time 47.43 seconds +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 146 : data loading time 0.00 +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 146 : avg forward pass time 12.80 +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 146 : backward pass time 10.12 +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 146 : optimizer step time 1.57 +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 146-------------------- +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:04:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440317 41.584881 41.367382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.308594 67.519531 67.238281 +util 95.750000 99.000000 90.000000 +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 147 : avg iteration time 48.79 seconds +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 147 : data loading time 0.00 +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 147 : avg forward pass time 13.43 +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 147 : backward pass time 10.18 +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 147 : optimizer step time 1.57 +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 147-------------------- +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:05:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440314 41.584881 41.367371 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.420898 67.910156 67.257812 +util 95.750000 99.000000 92.000000 +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 148 : avg iteration time 46.15 seconds +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 148 : data loading time 0.00 +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 148 : avg forward pass time 12.17 +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 148 : backward pass time 10.12 +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 148 : optimizer step time 1.56 +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 148-------------------- +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:06:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440318 41.584881 41.367378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499023 67.871094 67.375000 +util 94.750000 99.000000 88.000000 +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 149 : avg iteration time 43.49 seconds +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 149 : data loading time 0.00 +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 149 : avg forward pass time 10.83 +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 149 : backward pass time 10.14 +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 149 : optimizer step time 1.58 +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 149-------------------- +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:06:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440373 41.584881 41.367378 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.642578 107.570312 +util 67.000000 72.000000 60.000000 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 150 : avg iteration time 161.43 seconds +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 150 : data loading time 0.00 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 150 : avg forward pass time 72.09 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 150 : backward pass time 8.43 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 150 : optimizer step time 0.40 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 150-------------------- +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7061 iter count: 1.0 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7061 iter count: 1.0 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7061 iter count: 1.0 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7061iter count: 1.0 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:09:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440378 41.584881 41.367378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.764160 67.929688 67.708984 +util 93.500000 100.000000 89.000000 +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 151 : avg iteration time 47.43 seconds +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 151 : data loading time 0.00 +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 151 : avg forward pass time 12.76 +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 151 : backward pass time 10.14 +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 151 : optimizer step time 1.57 +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 151-------------------- +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:10:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440373 41.584881 41.367378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.749512 67.929688 67.689453 +util 94.500000 100.000000 88.000000 +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 152 : avg iteration time 48.66 seconds +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 152 : data loading time 0.00 +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 152 : avg forward pass time 13.40 +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 152 : backward pass time 10.14 +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 152 : optimizer step time 1.57 +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 152-------------------- +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:11:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440407 41.584881 41.367378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.705566 67.708984 67.695312 +util 93.500000 98.000000 88.000000 +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 153 : avg iteration time 46.26 seconds +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 153 : data loading time 0.00 +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 153 : avg forward pass time 12.17 +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 153 : backward pass time 10.12 +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 153 : optimizer step time 1.60 +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 153-------------------- +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:12:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440433 41.584881 41.367378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.588379 67.753906 67.533203 +util 93.500000 99.000000 87.000000 +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 154 : avg iteration time 43.52 seconds +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 154 : data loading time 0.00 +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 154 : avg forward pass time 10.83 +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 154 : backward pass time 10.13 +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 154 : optimizer step time 1.58 +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 154-------------------- +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:12:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440493 41.585117 41.367378 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.687500 107.583984 +util 61.500000 67.000000 53.000000 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 155 : avg iteration time 158.25 seconds +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 155 : data loading time 0.00 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 155 : avg forward pass time 70.51 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 155 : backward pass time 8.45 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 155 : optimizer step time 0.40 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 155-------------------- +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6963 iter count: 1.0 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6963 iter count: 1.0 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6963 iter count: 1.0 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6963iter count: 1.0 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:15:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440426 41.584846 41.367378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.212402 67.246094 67.201172 +util 95.500000 99.000000 92.000000 +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 156 : avg iteration time 49.94 seconds +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 156 : data loading time 0.00 +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 156 : avg forward pass time 14.02 +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 156 : backward pass time 10.13 +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 156 : optimizer step time 1.57 +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 156-------------------- +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:16:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440442 41.584846 41.367432 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.046387 67.226562 66.986328 +util 94.000000 99.000000 87.000000 +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 157 : avg iteration time 46.27 seconds +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 157 : data loading time 0.00 +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 157 : avg forward pass time 12.21 +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 157 : backward pass time 10.15 +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 157 : optimizer step time 1.57 +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 157-------------------- +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:17:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440438 41.584846 41.367432 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.222168 67.519531 67.123047 +util 96.000000 100.000000 90.000000 +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 158 : avg iteration time 46.03 seconds +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 158 : data loading time 0.00 +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 158 : avg forward pass time 12.09 +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 158 : backward pass time 10.16 +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 158 : optimizer step time 1.55 +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 158-------------------- +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:17:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440496 41.585079 41.367432 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.363770 67.734375 67.240234 +util 95.750000 100.000000 92.000000 +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 159 : avg iteration time 46.10 seconds +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 159 : data loading time 0.00 +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 159 : avg forward pass time 12.13 +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 159 : backward pass time 10.12 +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 159 : optimizer step time 1.57 +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 159-------------------- +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:18:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440366 41.584534 41.367435 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.667969 107.564453 +util 62.250000 67.000000 55.000000 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 160 : avg iteration time 163.89 seconds +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 160 : data loading time 0.00 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 160 : avg forward pass time 73.33 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 160 : backward pass time 8.43 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 160 : optimizer step time 0.42 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 160-------------------- +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7178 iter count: 1.0 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7178 iter count: 1.0 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7178 iter count: 1.0 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7178iter count: 1.0 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:21:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440381 41.584557 41.367447 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.284180 67.539062 67.199219 +util 95.500000 99.000000 92.000000 +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 161 : avg iteration time 44.91 seconds +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 161 : data loading time 0.00 +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 161 : avg forward pass time 11.53 +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 161 : backward pass time 10.12 +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 161 : optimizer step time 1.57 +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 161-------------------- +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:22:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440378 41.584534 41.367458 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.254883 67.890625 67.042969 +util 94.500000 100.000000 91.000000 +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 162 : avg iteration time 45.98 seconds +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 162 : data loading time 0.00 +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 162 : avg forward pass time 12.09 +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 162 : backward pass time 10.12 +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 162 : optimizer step time 1.56 +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 162-------------------- +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:22:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440393 41.584568 41.367470 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.890625 67.199219 +util 95.750000 99.000000 90.000000 +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 163 : avg iteration time 46.13 seconds +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 163 : data loading time 0.00 +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 163 : avg forward pass time 12.14 +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 163 : backward pass time 10.13 +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 163 : optimizer step time 1.56 +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 163-------------------- +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:23:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440552 41.585205 41.367458 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 68.164062 67.179688 +util 93.250000 99.000000 89.000000 +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 164 : avg iteration time 46.25 seconds +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 164 : data loading time 0.00 +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 164 : avg forward pass time 12.20 +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 164 : backward pass time 10.13 +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 164 : optimizer step time 1.58 +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 164-------------------- +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:24:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440292 41.584152 41.367481 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.746582 108.351562 107.544922 +util 63.750000 69.000000 57.000000 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 165 : avg iteration time 161.12 seconds +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 165 : data loading time 0.00 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 165 : avg forward pass time 71.91 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 165 : backward pass time 8.43 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 165 : optimizer step time 0.39 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 165-------------------- +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6914 iter count: 1.0 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6914 iter count: 1.0 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6914 iter count: 1.0 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6914iter count: 1.0 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:27:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:27:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440347 41.584290 41.367493 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.494141 67.304688 +util 94.500000 99.000000 88.000000 +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 166 : avg iteration time 49.99 seconds +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 166 : data loading time 0.00 +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 166 : avg forward pass time 14.03 +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 166 : backward pass time 10.11 +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 166 : optimizer step time 1.58 +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 166-------------------- +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:27:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440411 41.584507 41.367496 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.344238 67.357422 67.304688 +util 96.000000 100.000000 90.000000 +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 167 : avg iteration time 48.83 seconds +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 167 : data loading time 0.00 +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 167 : avg forward pass time 13.50 +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 167 : backward pass time 10.14 +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 167 : optimizer step time 1.56 +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 167-------------------- +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:28:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:29:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440524 41.584999 41.367493 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.563965 67.832031 67.474609 +util 95.250000 99.000000 89.000000 +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 168 : avg iteration time 46.19 seconds +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 168 : data loading time 0.00 +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 168 : avg forward pass time 12.16 +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 168 : backward pass time 10.13 +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 168 : optimizer step time 1.58 +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 168-------------------- +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:29:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440527 41.584976 41.367496 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.686035 67.734375 67.669922 +util 95.250000 99.000000 92.000000 +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 169 : avg iteration time 46.10 seconds +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 169 : data loading time 0.00 +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 169 : avg forward pass time 12.12 +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 169 : backward pass time 10.13 +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 169 : optimizer step time 1.58 +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 169-------------------- +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:30:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440454 41.584698 41.367496 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.746094 107.486328 +util 65.250000 72.000000 62.000000 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 170 : avg iteration time 161.41 seconds +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 170 : data loading time 0.00 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 170 : avg forward pass time 72.07 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 170 : backward pass time 8.43 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 170 : optimizer step time 0.40 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 170-------------------- +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7051 iter count: 1.0 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7051 iter count: 1.0 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7051 iter count: 1.0 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7051iter count: 1.0 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:32:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440459 41.584698 41.367519 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.890625 67.337891 +util 94.250000 99.000000 91.000000 +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 171 : avg iteration time 47.18 seconds +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 171 : data loading time 0.00 +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 171 : avg forward pass time 12.65 +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 171 : backward pass time 10.14 +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 171 : optimizer step time 1.58 +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 171-------------------- +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:33:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440473 41.584743 41.367504 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.485840 67.871094 67.357422 +util 95.750000 100.000000 92.000000 +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 172 : avg iteration time 46.22 seconds +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 172 : data loading time 0.00 +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 172 : avg forward pass time 12.18 +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 172 : backward pass time 10.15 +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 172 : optimizer step time 1.58 +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 172-------------------- +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:34:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440454 41.584698 41.367496 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.383301 67.871094 67.220703 +util 95.500000 100.000000 89.000000 +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 173 : avg iteration time 46.10 seconds +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 173 : data loading time 0.00 +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 173 : avg forward pass time 12.08 +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 173 : backward pass time 10.15 +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 173 : optimizer step time 1.56 +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 173-------------------- +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:35:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440455 41.584698 41.367496 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.471191 67.871094 67.337891 +util 93.250000 99.000000 89.000000 +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 174 : avg iteration time 46.26 seconds +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 174 : data loading time 0.00 +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 174 : avg forward pass time 12.20 +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 174 : backward pass time 10.14 +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 174 : optimizer step time 1.58 +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 174-------------------- +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:35:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440287 41.584011 41.367496 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.687500 107.544922 +util 66.000000 72.000000 62.000000 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 175 : avg iteration time 158.65 seconds +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 175 : data loading time 0.00 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 175 : avg forward pass time 70.72 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 175 : backward pass time 8.41 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 175 : optimizer step time 0.40 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 175-------------------- +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6934 iter count: 1.0 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6934 iter count: 1.0 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6934 iter count: 1.0 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6934iter count: 1.0 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:38:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440288 41.584003 41.367496 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.274414 67.500000 67.199219 +util 95.750000 100.000000 92.000000 +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 176 : avg iteration time 47.38 seconds +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 176 : data loading time 0.00 +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 176 : avg forward pass time 12.76 +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 176 : backward pass time 10.16 +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 176 : optimizer step time 1.56 +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 176-------------------- +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:39:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440467 41.584740 41.367500 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.284180 67.539062 67.199219 +util 96.500000 100.000000 93.000000 +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 177 : avg iteration time 46.21 seconds +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 177 : data loading time 0.00 +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 177 : avg forward pass time 12.19 +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 177 : backward pass time 10.14 +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 177 : optimizer step time 1.57 +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 177-------------------- +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:40:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440466 41.584728 41.367508 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.890625 67.140625 +util 94.750000 99.000000 89.000000 +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 178 : avg iteration time 43.62 seconds +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 178 : data loading time 0.00 +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 178 : avg forward pass time 10.88 +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 178 : backward pass time 10.13 +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 178 : optimizer step time 1.58 +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 178-------------------- +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:40:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440475 41.584747 41.367519 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.264648 67.695312 67.121094 +util 93.250000 99.000000 89.000000 +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 179 : avg iteration time 48.74 seconds +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 179 : data loading time 0.00 +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 179 : avg forward pass time 13.43 +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 179 : backward pass time 10.13 +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 179 : optimizer step time 1.59 +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 179-------------------- +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:41:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440147 41.583424 41.367519 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.541504 107.707031 107.486328 +util 68.750000 74.000000 61.000000 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 180 : avg iteration time 163.99 seconds +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 180 : data loading time 0.00 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 180 : avg forward pass time 73.37 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 180 : backward pass time 8.45 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 180 : optimizer step time 0.41 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 180-------------------- +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7354 iter count: 1.0 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7354 iter count: 1.0 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7354 iter count: 1.0 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7354iter count: 1.0 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440179 41.583447 41.367542 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.656250 67.376953 +util 95.000000 99.000000 89.000000 +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 181 : avg iteration time 47.28 seconds +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 181 : data loading time 0.00 +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 181 : avg forward pass time 12.70 +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 181 : backward pass time 10.16 +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 181 : optimizer step time 1.57 +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 181-------------------- +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:45:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440351 41.584137 41.367519 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.617676 67.812500 67.552734 +util 94.000000 99.000000 87.000000 +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 182 : avg iteration time 46.23 seconds +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 182 : data loading time 0.00 +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 182 : avg forward pass time 12.18 +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 182 : backward pass time 10.13 +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 182 : optimizer step time 1.58 +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 182-------------------- +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:46:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440363 41.584137 41.367565 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.510254 67.851562 67.396484 +util 95.750000 100.000000 92.000000 +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 183 : avg iteration time 48.93 seconds +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 183 : data loading time 0.00 +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 183 : avg forward pass time 13.53 +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 183 : backward pass time 10.14 +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 183 : optimizer step time 1.57 +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 183-------------------- +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:46:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440410 41.584362 41.367527 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.510254 67.851562 67.396484 +util 94.500000 99.000000 88.000000 +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 184 : avg iteration time 43.55 seconds +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 184 : data loading time 0.00 +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 184 : avg forward pass time 10.85 +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 184 : backward pass time 10.13 +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 184 : optimizer step time 1.57 +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 184-------------------- +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:47:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440196 41.583473 41.367580 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.609375 107.564453 +util 61.750000 68.000000 58.000000 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 185 : avg iteration time 158.63 seconds +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 185 : data loading time 0.00 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 185 : avg forward pass time 70.67 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 185 : backward pass time 8.42 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 185 : optimizer step time 0.41 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 185-------------------- +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6465 iter count: 1.0 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6465 iter count: 1.0 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6465 iter count: 1.0 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6465iter count: 1.0 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:50:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440203 41.583477 41.367580 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.523438 67.550781 67.441406 +util 93.500000 100.000000 89.000000 +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 186 : avg iteration time 49.79 seconds +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 186 : data loading time 0.00 +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 186 : avg forward pass time 13.97 +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 186 : backward pass time 10.14 +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 186 : optimizer step time 1.57 +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 186-------------------- +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:51:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440260 41.583714 41.367558 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.069336 67.441406 66.945312 +util 93.250000 99.000000 89.000000 +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 187 : avg iteration time 44.01 seconds +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 187 : data loading time 0.00 +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 187 : avg forward pass time 11.08 +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 187 : backward pass time 10.13 +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 187 : optimizer step time 1.57 +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 187-------------------- +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:51:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440625 41.585159 41.367588 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.279297 67.460938 67.218750 +util 95.750000 99.000000 93.000000 +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 188 : avg iteration time 44.23 seconds +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 188 : data loading time 0.00 +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 188 : avg forward pass time 11.16 +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 188 : backward pass time 10.13 +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 188 : optimizer step time 1.57 +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 188-------------------- +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:52:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440679 41.585411 41.367569 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.433594 67.304688 +util 96.500000 100.000000 91.000000 +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 189 : avg iteration time 48.85 seconds +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 189 : data loading time 0.00 +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 189 : avg forward pass time 13.50 +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 189 : backward pass time 10.13 +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 189 : optimizer step time 1.57 +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 189-------------------- +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:53:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440314 41.583889 41.367592 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.707031 107.505859 +util 65.500000 70.000000 58.000000 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 190 : avg iteration time 163.65 seconds +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 190 : data loading time 0.00 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 190 : avg forward pass time 73.17 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 190 : backward pass time 8.45 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 190 : optimizer step time 0.39 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 190-------------------- +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:56:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440322 41.583908 41.367615 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.383301 67.402344 67.376953 +util 93.500000 98.000000 86.000000 +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 191 : avg iteration time 49.76 seconds +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 191 : data loading time 0.00 +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 191 : avg forward pass time 13.96 +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 191 : backward pass time 10.14 +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 191 : optimizer step time 1.58 +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 191-------------------- +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:56:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440428 41.584347 41.367592 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.549316 67.714844 67.494141 +util 94.500000 99.000000 88.000000 +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 192 : avg iteration time 43.28 seconds +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 192 : data loading time 0.00 +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 192 : avg forward pass time 10.73 +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 192 : backward pass time 10.12 +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 192 : optimizer step time 1.57 +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 192-------------------- +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:57:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440729 41.585575 41.367592 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.563965 67.714844 67.513672 +util 95.750000 99.000000 90.000000 +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 193 : avg iteration time 48.76 seconds +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 193 : data loading time 0.00 +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 193 : avg forward pass time 13.45 +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 193 : backward pass time 10.14 +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 193 : optimizer step time 1.58 +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 193-------------------- +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:58:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440769 41.585575 41.367599 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.329590 67.357422 67.246094 +util 94.750000 99.000000 88.000000 +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 194 : avg iteration time 48.58 seconds +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 194 : data loading time 0.00 +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 194 : avg forward pass time 13.36 +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 194 : backward pass time 10.15 +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 194 : optimizer step time 1.57 +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 194-------------------- +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 22:59:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440858 41.585583 41.367966 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.531738 107.667969 107.486328 +util 62.250000 67.000000 54.000000 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 195 : avg iteration time 155.58 seconds +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 195 : data loading time 0.00 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 195 : avg forward pass time 69.19 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 195 : backward pass time 8.38 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 195 : optimizer step time 0.40 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 195-------------------- +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6221 iter count: 1.0 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6221 iter count: 1.0 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6221 iter count: 1.0 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6221iter count: 1.0 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:01:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440606 41.584572 41.367977 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.393066 67.441406 67.376953 +util 93.500000 98.000000 86.000000 +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 196 : avg iteration time 49.80 seconds +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 196 : data loading time 0.00 +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 196 : avg forward pass time 13.97 +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 196 : backward pass time 10.15 +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 196 : optimizer step time 1.58 +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 196-------------------- +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:02:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440609 41.584572 41.367989 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.421875 67.396484 +util 93.500000 99.000000 87.000000 +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 197 : avg iteration time 48.75 seconds +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 197 : data loading time 0.00 +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 197 : avg forward pass time 13.45 +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 197 : backward pass time 10.13 +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 197 : optimizer step time 1.58 +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 197-------------------- +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:03:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440615 41.584572 41.367989 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.388184 67.421875 67.376953 +util 93.750000 100.000000 90.000000 +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 198 : avg iteration time 46.16 seconds +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 198 : data loading time 0.00 +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 198 : avg forward pass time 12.17 +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 198 : backward pass time 10.14 +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 198 : optimizer step time 1.56 +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 198-------------------- +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:04:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440615 41.584583 41.367989 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.510254 67.533203 67.441406 +util 96.750000 99.000000 92.000000 +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 199 : avg iteration time 45.95 seconds +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 199 : data loading time 0.00 +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 199 : avg forward pass time 12.06 +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 199 : backward pass time 10.13 +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 199 : optimizer step time 1.57 +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 199-------------------- +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:04:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440615 41.584583 41.367989 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.667969 107.544922 +util 62.500000 69.000000 58.000000 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 200 : avg iteration time 163.45 seconds +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 200 : data loading time 0.00 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 200 : avg forward pass time 73.14 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 200 : backward pass time 8.40 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 200 : optimizer step time 0.40 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 200-------------------- +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6709 iter count: 1.0 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6709 iter count: 1.0 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6709 iter count: 1.0 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6709iter count: 1.0 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 23:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 6, 2026 - 23:07:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.40 seconds +[May 6, 2026 - 23:07:57 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 200 +[May 6, 2026 - 23:07:57 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000200 +[May 6, 2026 - 23:08:15 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 200 +[May 6, 2026 - 23:08:15 | INFO | fastgen.trainer:validate:358 ] Validation iteration 200 +[May 6, 2026 - 23:08:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 6, 2026 - 23:08:21 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.375 +[May 6, 2026 - 23:08:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:08:26 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.142 +[May 6, 2026 - 23:08:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:08:31 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.694 +[May 6, 2026 - 23:08:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:08:36 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.404 +[May 6, 2026 - 23:08:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:08:41 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.364 +[May 6, 2026 - 23:08:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:08:46 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.009 +[May 6, 2026 - 23:08:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:08:51 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.351 +[May 6, 2026 - 23:08:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:08:56 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.295 +[May 6, 2026 - 23:08:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:08:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:09:01 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.981 +[May 6, 2026 - 23:09:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 6, 2026 - 23:09:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 6, 2026 - 23:09:06 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.776 +[May 6, 2026 - 23:09:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 200-------------------- +[May 6, 2026 - 23:09:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 6, 2026 - 23:09:22 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 200 +[May 6, 2026 - 23:09:22 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.439 (n=10) +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442726 41.591930 41.368088 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777344 68.222656 67.628906 +util 95.750000 100.000000 89.000000 +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 201 : avg iteration time 145.13 seconds +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 201 : data loading time 0.00 +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 201 : avg forward pass time 11.72 +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 201 : backward pass time 10.13 +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 201 : optimizer step time 1.57 +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 201-------------------- +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:10:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442393 41.590618 41.368088 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.982422 68.222656 67.902344 +util 97.000000 100.000000 92.000000 +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 202 : avg iteration time 49.02 seconds +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 202 : data loading time 0.00 +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 202 : avg forward pass time 13.55 +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 202 : backward pass time 10.19 +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 202 : optimizer step time 1.57 +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 202-------------------- +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:10:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442406 41.590618 41.368114 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.967773 68.222656 67.882812 +util 94.500000 99.000000 87.000000 +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 203 : avg iteration time 46.14 seconds +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 203 : data loading time 0.00 +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 203 : avg forward pass time 12.12 +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 203 : backward pass time 10.15 +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 203 : optimizer step time 1.58 +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 203-------------------- +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:11:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441955 41.588802 41.368118 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.938477 68.222656 67.843750 +util 93.500000 99.000000 90.000000 +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 204 : avg iteration time 43.50 seconds +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 204 : data loading time 0.00 +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 204 : avg forward pass time 10.82 +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 204 : backward pass time 10.15 +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 204 : optimizer step time 1.57 +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 204-------------------- +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:12:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441638 41.587528 41.368134 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.083496 108.228516 107.648438 +util 64.500000 69.000000 58.000000 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 205 : avg iteration time 161.08 seconds +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 205 : data loading time 0.00 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 205 : avg forward pass time 71.91 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 205 : backward pass time 8.45 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 205 : optimizer step time 0.40 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 205-------------------- +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6406 iter count: 1.0 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6406 iter count: 1.0 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6406 iter count: 1.0 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6406iter count: 1.0 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:15:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441639 41.587547 41.368118 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.323242 67.335938 67.285156 +util 96.250000 100.000000 91.000000 +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 206 : avg iteration time 47.21 seconds +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 206 : data loading time 0.00 +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 206 : avg forward pass time 12.68 +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 206 : backward pass time 10.14 +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 206 : optimizer step time 1.58 +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 206-------------------- +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:15:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441721 41.587833 41.368118 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.480469 67.277344 +util 93.000000 99.000000 87.000000 +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 207 : avg iteration time 45.95 seconds +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 207 : data loading time 0.00 +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 207 : avg forward pass time 12.04 +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 207 : backward pass time 10.15 +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 207 : optimizer step time 1.57 +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 207-------------------- +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:16:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441717 41.587833 41.368118 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.318359 67.441406 67.277344 +util 97.000000 100.000000 92.000000 +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 208 : avg iteration time 46.12 seconds +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 208 : data loading time 0.00 +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 208 : avg forward pass time 12.14 +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 208 : backward pass time 10.12 +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 208 : optimizer step time 1.57 +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 208-------------------- +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:17:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441734 41.587856 41.368187 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.186523 67.265625 67.160156 +util 96.000000 100.000000 90.000000 +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 209 : avg iteration time 48.72 seconds +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 209 : data loading time 0.00 +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 209 : avg forward pass time 13.43 +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 209 : backward pass time 10.14 +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 209 : optimizer step time 1.57 +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 209-------------------- +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:18:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441476 41.586800 41.368187 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.648438 107.603516 +util 62.500000 69.000000 59.000000 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 210 : avg iteration time 158.13 seconds +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 210 : data loading time 0.00 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 210 : avg forward pass time 70.47 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 210 : backward pass time 8.42 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 210 : optimizer step time 0.40 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 210-------------------- +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6504 iter count: 1.0 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6504 iter count: 1.0 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6504 iter count: 1.0 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6504iter count: 1.0 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:20:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441473 41.586796 41.368195 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.441406 67.335938 +util 97.250000 100.000000 92.000000 +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 211 : avg iteration time 47.15 seconds +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 211 : data loading time 0.00 +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 211 : avg forward pass time 12.65 +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 211 : backward pass time 10.13 +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 211 : optimizer step time 1.58 +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 211-------------------- +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:21:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441534 41.587048 41.368195 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.406250 67.558594 67.355469 +util 93.750000 99.000000 90.000000 +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 212 : avg iteration time 48.64 seconds +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 212 : data loading time 0.00 +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 212 : avg forward pass time 13.40 +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 212 : backward pass time 10.14 +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 212 : optimizer step time 1.57 +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 212-------------------- +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:22:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441544 41.587025 41.368202 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.382812 67.355469 +util 94.000000 99.000000 90.000000 +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 213 : avg iteration time 43.40 seconds +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 213 : data loading time 0.00 +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 213 : avg forward pass time 10.76 +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 213 : backward pass time 10.14 +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 213 : optimizer step time 1.58 +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 213-------------------- +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:23:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441582 41.587231 41.368195 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.367188 67.402344 67.355469 +util 92.250000 99.000000 88.000000 +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 214 : avg iteration time 48.77 seconds +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 214 : data loading time 0.00 +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 214 : avg forward pass time 13.48 +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 214 : backward pass time 10.13 +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 214 : optimizer step time 1.56 +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 214-------------------- +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:24:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441167 41.585548 41.368195 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.648438 107.525391 +util 64.250000 69.000000 59.000000 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 215 : avg iteration time 161.09 seconds +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 215 : data loading time 0.00 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 215 : avg forward pass time 71.88 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 215 : backward pass time 8.48 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 215 : optimizer step time 0.40 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 215-------------------- +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6787 iter count: 1.0 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6787 iter count: 1.0 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6787 iter count: 1.0 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6787iter count: 1.0 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:26:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441168 41.585537 41.368210 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412598 67.460938 67.396484 +util 94.500000 99.000000 91.000000 +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 216 : avg iteration time 47.15 seconds +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 216 : data loading time 0.00 +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 216 : avg forward pass time 12.62 +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 216 : backward pass time 10.17 +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 216 : optimizer step time 1.57 +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 216-------------------- +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:27:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441237 41.585785 41.368240 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412598 67.519531 67.376953 +util 95.250000 100.000000 92.000000 +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 217 : avg iteration time 46.21 seconds +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 217 : data loading time 0.00 +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 217 : avg forward pass time 12.16 +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 217 : backward pass time 10.14 +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 217 : optimizer step time 1.58 +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 217-------------------- +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:28:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441514 41.586903 41.368229 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412598 67.519531 67.376953 +util 93.750000 99.000000 90.000000 +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 218 : avg iteration time 48.58 seconds +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 218 : data loading time 0.00 +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 218 : avg forward pass time 13.36 +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 218 : backward pass time 10.15 +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 218 : optimizer step time 1.57 +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 218-------------------- +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:29:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441607 41.587257 41.368233 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412598 67.519531 67.376953 +util 94.500000 100.000000 91.000000 +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 219 : avg iteration time 46.21 seconds +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 219 : data loading time 0.00 +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 219 : avg forward pass time 12.19 +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 219 : backward pass time 10.14 +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 219 : optimizer step time 1.56 +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 219-------------------- +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:29:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441335 41.586151 41.368237 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.804688 107.564453 +util 64.000000 69.000000 58.000000 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 220 : avg iteration time 160.45 seconds +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 220 : data loading time 0.00 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 220 : avg forward pass time 71.62 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 220 : backward pass time 8.41 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 220 : optimizer step time 0.39 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 220-------------------- +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6436 iter count: 1.0 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6436 iter count: 1.0 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6436 iter count: 1.0 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6436iter count: 1.0 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:32:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441338 41.586151 41.368237 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.578125 67.375000 +util 95.250000 99.000000 90.000000 +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 221 : avg iteration time 44.81 seconds +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 221 : data loading time 0.00 +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 221 : avg forward pass time 11.46 +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 221 : backward pass time 10.15 +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 221 : optimizer step time 1.58 +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 221-------------------- +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:33:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441349 41.586174 41.368237 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.538086 68.144531 67.335938 +util 96.500000 99.000000 91.000000 +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 222 : avg iteration time 48.48 seconds +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 222 : data loading time 0.00 +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 222 : avg forward pass time 13.32 +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 222 : backward pass time 10.14 +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 222 : optimizer step time 1.56 +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 222-------------------- +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:34:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441470 41.586655 41.368248 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396484 67.636719 67.316406 +util 94.750000 99.000000 91.000000 +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 223 : avg iteration time 48.68 seconds +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 223 : data loading time 0.00 +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 223 : avg forward pass time 13.40 +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 223 : backward pass time 10.17 +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 223 : optimizer step time 1.57 +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 223-------------------- +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:34:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:35:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441531 41.586903 41.368248 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.636719 67.355469 +util 94.750000 99.000000 88.000000 +[May 6, 2026 - 23:35:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 224 : avg iteration time 43.28 seconds +[May 6, 2026 - 23:35:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 224 : data loading time 0.00 +[May 6, 2026 - 23:35:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 224 : avg forward pass time 10.72 +[May 6, 2026 - 23:35:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 224 : backward pass time 10.14 +[May 6, 2026 - 23:35:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 224 : optimizer step time 1.56 +[May 6, 2026 - 23:35:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 224-------------------- +[May 6, 2026 - 23:35:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:35:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:35:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:35:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441066 41.585014 41.368248 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.648438 107.623047 +util 65.250000 70.000000 60.000000 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 225 : avg iteration time 166.17 seconds +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 225 : data loading time 0.00 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 225 : avg forward pass time 74.45 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 225 : backward pass time 8.42 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 225 : optimizer step time 0.40 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 225-------------------- +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6201 iter count: 1.0 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6201 iter count: 1.0 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6201 iter count: 1.0 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6201iter count: 1.0 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:38:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441089 41.585068 41.368248 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.513672 68.105469 67.316406 +util 95.250000 100.000000 88.000000 +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 226 : avg iteration time 49.83 seconds +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 226 : data loading time 0.00 +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 226 : avg forward pass time 14.00 +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 226 : backward pass time 10.14 +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 226 : optimizer step time 1.57 +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 226-------------------- +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:39:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441136 41.585270 41.368248 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.547852 68.183594 67.335938 +util 94.250000 99.000000 91.000000 +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 227 : avg iteration time 48.90 seconds +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 227 : data loading time 0.00 +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 227 : avg forward pass time 13.53 +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 227 : backward pass time 10.13 +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 227 : optimizer step time 1.58 +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 227-------------------- +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:40:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441500 41.586723 41.368248 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.567383 68.261719 67.335938 +util 95.500000 100.000000 90.000000 +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 228 : avg iteration time 43.46 seconds +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 228 : data loading time 0.00 +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 228 : avg forward pass time 10.81 +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 228 : backward pass time 10.14 +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 228 : optimizer step time 1.57 +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 228-------------------- +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:40:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441565 41.586975 41.368256 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.586914 68.281250 67.355469 +util 96.000000 99.000000 90.000000 +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 229 : avg iteration time 48.67 seconds +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 229 : data loading time 0.00 +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 229 : avg forward pass time 13.41 +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 229 : backward pass time 10.12 +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 229 : optimizer step time 1.56 +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 229-------------------- +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:41:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441638 41.587234 41.368248 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.746094 107.544922 +util 64.000000 70.000000 60.000000 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 230 : avg iteration time 158.27 seconds +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 230 : data loading time 0.00 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 230 : avg forward pass time 70.52 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 230 : backward pass time 8.42 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 230 : optimizer step time 0.40 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 230-------------------- +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6543 iter count: 1.0 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6543 iter count: 1.0 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6543 iter count: 1.0 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6543iter count: 1.0 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:44:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441266 41.585766 41.368248 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.515137 67.871094 67.396484 +util 95.500000 99.000000 89.000000 +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 231 : avg iteration time 49.91 seconds +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 231 : data loading time 0.00 +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 231 : avg forward pass time 14.04 +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 231 : backward pass time 10.12 +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 231 : optimizer step time 1.57 +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 231-------------------- +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:45:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441274 41.585793 41.368275 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.349121 67.441406 67.318359 +util 96.000000 100.000000 91.000000 +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 232 : avg iteration time 46.16 seconds +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 232 : data loading time 0.00 +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 232 : avg forward pass time 12.16 +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 232 : backward pass time 10.13 +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 232 : optimizer step time 1.58 +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 232-------------------- +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:45:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441267 41.585766 41.368275 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.968750 67.318359 +util 94.250000 100.000000 90.000000 +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 233 : avg iteration time 43.32 seconds +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 233 : data loading time 0.00 +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 233 : avg forward pass time 10.74 +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 233 : backward pass time 10.13 +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 233 : optimizer step time 1.57 +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 233-------------------- +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:46:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441268 41.585766 41.368275 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.427246 67.753906 67.318359 +util 94.250000 100.000000 90.000000 +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 234 : avg iteration time 46.00 seconds +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 234 : data loading time 0.00 +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 234 : avg forward pass time 12.07 +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 234 : backward pass time 10.13 +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 234 : optimizer step time 1.57 +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 234-------------------- +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:47:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441332 41.586002 41.368286 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.746094 107.564453 +util 63.000000 68.000000 55.000000 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 235 : avg iteration time 158.49 seconds +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 235 : data loading time 0.00 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 235 : avg forward pass time 70.64 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 235 : backward pass time 8.39 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 235 : optimizer step time 0.40 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 235-------------------- +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6221 iter count: 1.0 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6221 iter count: 1.0 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6221 iter count: 1.0 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6221iter count: 1.0 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:49:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441300 41.585888 41.368286 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.391602 67.500000 67.355469 +util 95.250000 100.000000 89.000000 +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 236 : avg iteration time 44.67 seconds +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 236 : data loading time 0.00 +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 236 : avg forward pass time 11.40 +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 236 : backward pass time 10.14 +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 236 : optimizer step time 1.58 +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 236-------------------- +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:50:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441307 41.585896 41.368286 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.480469 67.375000 +util 95.750000 99.000000 91.000000 +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 237 : avg iteration time 43.51 seconds +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 237 : data loading time 0.00 +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 237 : avg forward pass time 10.82 +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 237 : backward pass time 10.16 +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 237 : optimizer step time 1.58 +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 237-------------------- +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:51:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441306 41.585884 41.368309 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.480469 67.375000 +util 95.250000 99.000000 89.000000 +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 238 : avg iteration time 48.64 seconds +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 238 : data loading time 0.00 +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 238 : avg forward pass time 13.39 +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 238 : backward pass time 10.15 +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 238 : optimizer step time 1.55 +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 238-------------------- +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:52:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441573 41.586876 41.368362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.519531 67.375000 +util 95.000000 99.000000 88.000000 +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 239 : avg iteration time 49.00 seconds +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 239 : data loading time 0.00 +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 239 : avg forward pass time 13.57 +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 239 : backward pass time 10.15 +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 239 : optimizer step time 1.57 +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 239-------------------- +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:53:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441216 41.585464 41.368362 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.751465 107.785156 107.740234 +util 64.250000 71.000000 60.000000 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 240 : avg iteration time 160.90 seconds +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 240 : data loading time 0.00 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 240 : avg forward pass time 71.83 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 240 : backward pass time 8.43 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 240 : optimizer step time 0.40 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 240-------------------- +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6875 iter count: 1.0 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6875 iter count: 1.0 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6875 iter count: 1.0 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6875iter count: 1.0 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:55:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441219 41.585464 41.368340 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.460938 67.355469 +util 95.000000 100.000000 91.000000 +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 241 : avg iteration time 47.40 seconds +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 241 : data loading time 0.00 +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 241 : avg forward pass time 12.76 +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 241 : backward pass time 10.16 +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 241 : optimizer step time 1.58 +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 241-------------------- +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:56:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441221 41.585484 41.368362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.441406 67.335938 +util 95.000000 99.000000 90.000000 +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 242 : avg iteration time 43.45 seconds +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 242 : data loading time 0.00 +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 242 : avg forward pass time 10.77 +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 242 : backward pass time 10.13 +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 242 : optimizer step time 1.57 +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 242-------------------- +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441225 41.585476 41.368362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.367188 67.460938 67.335938 +util 92.500000 98.000000 87.000000 +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 243 : avg iteration time 45.96 seconds +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 243 : data loading time 0.00 +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 243 : avg forward pass time 12.06 +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 243 : backward pass time 10.13 +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 243 : optimizer step time 1.57 +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 243-------------------- +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:57:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441423 41.586189 41.368343 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.352539 67.402344 67.335938 +util 92.750000 98.000000 88.000000 +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 244 : avg iteration time 43.40 seconds +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 244 : data loading time 0.00 +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 244 : avg forward pass time 10.74 +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 244 : backward pass time 10.21 +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 244 : optimizer step time 1.58 +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 244-------------------- +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 6, 2026 - 23:58:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441236 41.585403 41.368362 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.746094 107.505859 +util 63.500000 68.000000 58.000000 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 245 : avg iteration time 158.62 seconds +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 245 : data loading time 0.00 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 245 : avg forward pass time 70.68 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 245 : backward pass time 8.40 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 245 : optimizer step time 0.40 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 245-------------------- +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6553 iter count: 1.0 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6553 iter count: 1.0 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6553 iter count: 1.0 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6553iter count: 1.0 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:01:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441227 41.585415 41.368340 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.528320 67.988281 67.375000 +util 93.750000 99.000000 90.000000 +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 246 : avg iteration time 47.35 seconds +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 246 : data loading time 0.00 +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 246 : avg forward pass time 12.75 +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 246 : backward pass time 10.12 +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 246 : optimizer step time 1.57 +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 246-------------------- +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:02:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441233 41.585403 41.368362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.450195 67.851562 67.316406 +util 94.750000 99.000000 91.000000 +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 247 : avg iteration time 46.22 seconds +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 247 : data loading time 0.00 +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 247 : avg forward pass time 12.18 +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 247 : backward pass time 10.14 +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 247 : optimizer step time 1.59 +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 247-------------------- +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:02:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441236 41.585403 41.368362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 68.027344 67.316406 +util 93.000000 98.000000 89.000000 +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 248 : avg iteration time 48.62 seconds +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 248 : data loading time 0.00 +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 248 : avg forward pass time 13.38 +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 248 : backward pass time 10.13 +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 248 : optimizer step time 1.59 +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 248-------------------- +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:03:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441230 41.585403 41.368362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.812500 67.316406 +util 92.500000 99.000000 88.000000 +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 249 : avg iteration time 46.14 seconds +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 249 : data loading time 0.00 +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 249 : avg forward pass time 12.14 +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 249 : backward pass time 10.15 +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 249 : optimizer step time 1.57 +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 249-------------------- +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:04:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441177 41.585217 41.368340 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.648926 107.785156 107.603516 +util 67.500000 74.000000 63.000000 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 250 : avg iteration time 166.21 seconds +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 250 : data loading time 0.00 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 250 : avg forward pass time 74.43 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 250 : backward pass time 8.45 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 250 : optimizer step time 0.40 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 250-------------------- +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6367 iter count: 1.0 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6367 iter count: 1.0 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6367 iter count: 1.0 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6367iter count: 1.0 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:07:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441194 41.585217 41.368362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 68.027344 67.316406 +util 95.750000 99.000000 91.000000 +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 251 : avg iteration time 49.89 seconds +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 251 : data loading time 0.00 +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 251 : avg forward pass time 14.01 +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 251 : backward pass time 10.14 +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 251 : optimizer step time 1.58 +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 251-------------------- +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:08:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441249 41.585457 41.368340 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 68.046875 67.199219 +util 93.250000 98.000000 86.000000 +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 252 : avg iteration time 48.61 seconds +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 252 : data loading time 0.00 +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 252 : avg forward pass time 13.39 +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 252 : backward pass time 10.11 +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 252 : optimizer step time 1.57 +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 252-------------------- +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:08:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441249 41.585449 41.368366 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.474609 68.007812 67.296875 +util 96.500000 99.000000 92.000000 +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 253 : avg iteration time 46.00 seconds +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 253 : data loading time 0.00 +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 253 : avg forward pass time 12.06 +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 253 : backward pass time 10.12 +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 253 : optimizer step time 1.57 +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 253-------------------- +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:09:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441302 41.585686 41.368343 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.479492 68.027344 67.296875 +util 93.500000 99.000000 88.000000 +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 254 : avg iteration time 45.96 seconds +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 254 : data loading time 0.00 +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 254 : avg forward pass time 12.07 +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 254 : backward pass time 10.11 +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 254 : optimizer step time 1.56 +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 254-------------------- +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:10:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441244 41.585445 41.368351 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.726562 107.525391 +util 68.500000 73.000000 61.000000 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 255 : avg iteration time 160.93 seconds +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 255 : data loading time 0.00 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 255 : avg forward pass time 71.84 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 255 : backward pass time 8.41 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 255 : optimizer step time 0.40 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 255-------------------- +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6904 iter count: 1.0 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6904 iter count: 1.0 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6904 iter count: 1.0 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6904iter count: 1.0 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:13:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441252 41.585453 41.368351 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.474609 67.656250 67.414062 +util 94.750000 99.000000 91.000000 +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 256 : avg iteration time 47.45 seconds +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 256 : data loading time 0.00 +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 256 : avg forward pass time 12.82 +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 256 : backward pass time 10.11 +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 256 : optimizer step time 1.57 +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 256-------------------- +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:13:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441252 41.585445 41.368351 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 68.281250 67.179688 +util 93.250000 99.000000 89.000000 +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 257 : avg iteration time 43.37 seconds +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 257 : data loading time 0.00 +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 257 : avg forward pass time 10.75 +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 257 : backward pass time 10.14 +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 257 : optimizer step time 1.57 +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 257-------------------- +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:14:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441247 41.585449 41.368351 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.416016 68.066406 67.199219 +util 94.250000 100.000000 90.000000 +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 258 : avg iteration time 48.81 seconds +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 258 : data loading time 0.00 +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 258 : avg forward pass time 13.46 +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 258 : backward pass time 10.14 +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 258 : optimizer step time 1.56 +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 258-------------------- +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:15:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441307 41.585678 41.368362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.396484 68.046875 67.179688 +util 93.000000 98.000000 89.000000 +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 259 : avg iteration time 46.02 seconds +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 259 : data loading time 0.00 +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 259 : avg forward pass time 12.09 +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 259 : backward pass time 10.14 +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 259 : optimizer step time 1.55 +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 259-------------------- +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:16:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441089 41.584759 41.368366 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.521973 107.628906 107.486328 +util 62.250000 67.000000 57.000000 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 260 : avg iteration time 158.22 seconds +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 260 : data loading time 0.00 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 260 : avg forward pass time 70.49 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 260 : backward pass time 8.41 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 260 : optimizer step time 0.40 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 260-------------------- +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6436 iter count: 1.0 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6436 iter count: 1.0 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6436 iter count: 1.0 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6436iter count: 1.0 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:18:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441086 41.584751 41.368382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.460938 67.355469 +util 93.500000 99.000000 88.000000 +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 261 : avg iteration time 44.60 seconds +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 261 : data loading time 0.00 +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 261 : avg forward pass time 11.37 +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 261 : backward pass time 10.12 +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 261 : optimizer step time 1.58 +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 261-------------------- +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:19:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441088 41.584751 41.368366 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.445312 67.773438 67.335938 +util 94.250000 99.000000 89.000000 +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 262 : avg iteration time 48.61 seconds +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 262 : data loading time 0.00 +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 262 : avg forward pass time 13.39 +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 262 : backward pass time 10.15 +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 262 : optimizer step time 1.56 +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 262-------------------- +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:20:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441151 41.585003 41.368370 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.832031 67.335938 +util 94.750000 100.000000 90.000000 +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 263 : avg iteration time 43.36 seconds +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 263 : data loading time 0.00 +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 263 : avg forward pass time 10.76 +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 263 : backward pass time 10.12 +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 263 : optimizer step time 1.58 +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 263-------------------- +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:21:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441158 41.585003 41.368370 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.480469 67.335938 +util 92.250000 99.000000 87.000000 +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 264 : avg iteration time 46.36 seconds +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 264 : data loading time 0.00 +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 264 : avg forward pass time 12.26 +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 264 : backward pass time 10.14 +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 264 : optimizer step time 1.56 +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 264-------------------- +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:21:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440947 41.584118 41.368382 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.658691 107.707031 107.642578 +util 63.750000 69.000000 56.000000 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 265 : avg iteration time 157.93 seconds +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 265 : data loading time 0.00 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 265 : avg forward pass time 70.37 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 265 : backward pass time 8.43 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 265 : optimizer step time 0.40 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 265-------------------- +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6045 iter count: 1.0 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6045 iter count: 1.0 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6045 iter count: 1.0 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6045iter count: 1.0 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:24:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.440948 41.584118 41.368382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.513672 67.531250 67.460938 +util 97.250000 100.000000 92.000000 +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 266 : avg iteration time 47.45 seconds +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 266 : data loading time 0.00 +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 266 : avg forward pass time 12.76 +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 266 : backward pass time 10.22 +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 266 : optimizer step time 1.56 +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 266-------------------- +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:25:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441010 41.584370 41.368382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.601562 67.667969 67.402344 +util 95.250000 99.000000 91.000000 +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 267 : avg iteration time 45.92 seconds +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 267 : data loading time 0.00 +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 267 : avg forward pass time 12.03 +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 267 : backward pass time 10.14 +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 267 : optimizer step time 1.59 +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 267-------------------- +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:26:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441319 41.585617 41.368397 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.689453 67.785156 67.402344 +util 94.000000 100.000000 89.000000 +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 268 : avg iteration time 48.73 seconds +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 268 : data loading time 0.00 +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 268 : avg forward pass time 13.41 +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 268 : backward pass time 10.15 +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 268 : optimizer step time 1.59 +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 268-------------------- +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:26:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441321 41.585617 41.368382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.733398 67.843750 67.402344 +util 94.250000 100.000000 90.000000 +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 269 : avg iteration time 43.40 seconds +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 269 : data loading time 0.00 +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 269 : avg forward pass time 10.78 +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 269 : backward pass time 10.13 +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 269 : optimizer step time 1.57 +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 269-------------------- +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:27:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441270 41.585426 41.368385 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.746094 107.603516 +util 67.750000 72.000000 60.000000 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 270 : avg iteration time 158.35 seconds +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 270 : data loading time 0.00 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 270 : avg forward pass time 70.52 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 270 : backward pass time 8.44 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 270 : optimizer step time 0.40 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 270-------------------- +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6377 iter count: 1.0 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6377 iter count: 1.0 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6377 iter count: 1.0 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6377iter count: 1.0 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:30:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441183 41.585079 41.368382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.495605 67.910156 67.357422 +util 97.250000 99.000000 92.000000 +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 271 : avg iteration time 49.81 seconds +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 271 : data loading time 0.00 +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 271 : avg forward pass time 14.00 +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 271 : backward pass time 10.10 +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 271 : optimizer step time 1.56 +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 271-------------------- +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:31:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441177 41.585056 41.368382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.417480 67.714844 67.318359 +util 96.250000 100.000000 92.000000 +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 272 : avg iteration time 43.52 seconds +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 272 : data loading time 0.00 +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 272 : avg forward pass time 10.81 +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 272 : backward pass time 10.19 +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 272 : optimizer step time 1.56 +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 272-------------------- +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:31:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441184 41.585056 41.368397 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.490723 67.890625 67.357422 +util 93.500000 99.000000 88.000000 +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 273 : avg iteration time 48.76 seconds +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 273 : data loading time 0.00 +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 273 : avg forward pass time 13.47 +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 273 : backward pass time 10.13 +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 273 : optimizer step time 1.57 +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 273-------------------- +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:32:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441190 41.585056 41.368397 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.427246 67.753906 67.318359 +util 93.750000 99.000000 89.000000 +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 274 : avg iteration time 43.60 seconds +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 274 : data loading time 0.00 +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 274 : avg forward pass time 10.88 +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 274 : backward pass time 10.12 +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 274 : optimizer step time 1.56 +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 274-------------------- +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:33:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441269 41.585304 41.368477 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.642578 107.609375 +util 63.750000 69.000000 56.000000 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 275 : avg iteration time 161.47 seconds +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 275 : data loading time 0.00 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 275 : avg forward pass time 72.08 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 275 : backward pass time 8.42 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 275 : optimizer step time 0.40 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 275-------------------- +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6758 iter count: 1.0 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6758 iter count: 1.0 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6758 iter count: 1.0 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6758iter count: 1.0 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:36:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441188 41.584980 41.368477 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.427246 67.578125 67.376953 +util 95.500000 100.000000 88.000000 +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 276 : avg iteration time 49.80 seconds +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 276 : data loading time 0.00 +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 276 : avg forward pass time 13.99 +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 276 : backward pass time 10.14 +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 276 : optimizer step time 1.55 +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 276-------------------- +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:36:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441188 41.584980 41.368477 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.427246 67.578125 67.376953 +util 94.250000 100.000000 90.000000 +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 277 : avg iteration time 48.69 seconds +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 277 : data loading time 0.00 +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 277 : avg forward pass time 13.44 +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 277 : backward pass time 10.12 +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 277 : optimizer step time 1.57 +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 277-------------------- +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:37:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441195 41.584999 41.368477 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.597656 67.396484 +util 94.750000 99.000000 88.000000 +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 278 : avg iteration time 48.57 seconds +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 278 : data loading time 0.00 +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 278 : avg forward pass time 13.38 +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 278 : backward pass time 10.11 +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 278 : optimizer step time 1.56 +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 278-------------------- +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:38:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441196 41.585003 41.368477 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.500000 67.376953 +util 93.250000 100.000000 88.000000 +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 279 : avg iteration time 43.42 seconds +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 279 : data loading time 0.00 +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 279 : avg forward pass time 10.80 +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 279 : backward pass time 10.12 +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 279 : optimizer step time 1.57 +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 279-------------------- +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:39:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441201 41.585003 41.368488 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.746094 107.623047 +util 63.250000 68.000000 58.000000 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 280 : avg iteration time 163.46 seconds +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 280 : data loading time 0.00 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 280 : avg forward pass time 73.12 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 280 : backward pass time 8.46 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 280 : optimizer step time 0.40 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 280-------------------- +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6377 iter count: 1.0 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6377 iter count: 1.0 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6377 iter count: 1.0 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6377iter count: 1.0 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:41:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441171 41.584885 41.368492 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.388184 67.539062 67.337891 +util 95.250000 99.000000 91.000000 +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 281 : avg iteration time 44.62 seconds +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 281 : data loading time 0.00 +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 281 : avg forward pass time 11.39 +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 281 : backward pass time 10.14 +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 281 : optimizer step time 1.57 +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 281-------------------- +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:42:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441165 41.584866 41.368488 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.378418 67.500000 67.337891 +util 95.000000 100.000000 90.000000 +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 282 : avg iteration time 48.79 seconds +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 282 : data loading time 0.00 +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 282 : avg forward pass time 13.45 +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 282 : backward pass time 10.19 +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 282 : optimizer step time 1.55 +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 282-------------------- +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:43:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441175 41.584888 41.368504 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358887 67.480469 67.318359 +util 94.500000 99.000000 87.000000 +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 283 : avg iteration time 48.65 seconds +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 283 : data loading time 0.00 +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 283 : avg forward pass time 13.40 +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 283 : backward pass time 10.14 +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 283 : optimizer step time 1.56 +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 283-------------------- +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:44:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441171 41.584869 41.368488 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.368652 67.519531 67.318359 +util 93.250000 99.000000 89.000000 +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 284 : avg iteration time 46.09 seconds +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 284 : data loading time 0.00 +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 284 : avg forward pass time 12.11 +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 284 : backward pass time 10.14 +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 284 : optimizer step time 1.58 +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 284-------------------- +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:45:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441169 41.584869 41.368500 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.683105 107.701172 107.628906 +util 64.250000 67.000000 62.000000 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 285 : avg iteration time 160.90 seconds +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 285 : data loading time 0.00 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 285 : avg forward pass time 71.79 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 285 : backward pass time 8.44 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 285 : optimizer step time 0.40 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 285-------------------- +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6797 iter count: 1.0 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6797 iter count: 1.0 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6797 iter count: 1.0 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6797iter count: 1.0 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:47:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441170 41.584869 41.368488 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.388184 67.480469 67.357422 +util 97.250000 100.000000 92.000000 +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 286 : avg iteration time 44.87 seconds +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 286 : data loading time 0.00 +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 286 : avg forward pass time 11.50 +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 286 : backward pass time 10.14 +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 286 : optimizer step time 1.59 +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 286-------------------- +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:48:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441176 41.584892 41.368488 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.583496 68.144531 67.396484 +util 94.750000 99.000000 88.000000 +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 287 : avg iteration time 48.75 seconds +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 287 : data loading time 0.00 +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 287 : avg forward pass time 13.46 +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 287 : backward pass time 10.13 +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 287 : optimizer step time 1.57 +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 287-------------------- +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:49:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441181 41.584869 41.368500 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.597656 67.396484 +util 97.000000 100.000000 91.000000 +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 288 : avg iteration time 48.63 seconds +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 288 : data loading time 0.00 +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 288 : avg forward pass time 13.39 +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 288 : backward pass time 10.14 +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 288 : optimizer step time 1.58 +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 288-------------------- +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:50:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441358 41.585606 41.368500 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.422363 67.500000 67.396484 +util 93.000000 99.000000 88.000000 +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 289 : avg iteration time 48.85 seconds +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 289 : data loading time 0.00 +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 289 : avg forward pass time 13.50 +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 289 : backward pass time 10.14 +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 289 : optimizer step time 1.57 +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 289-------------------- +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:50:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441127 41.584682 41.368500 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.687988 107.707031 107.681641 +util 67.000000 75.000000 60.000000 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 290 : avg iteration time 163.81 seconds +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 290 : data loading time 0.00 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 290 : avg forward pass time 73.27 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 290 : backward pass time 8.41 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 290 : optimizer step time 0.40 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 290-------------------- +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6113 iter count: 1.0 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6113 iter count: 1.0 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6113 iter count: 1.0 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6113iter count: 1.0 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:53:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441128 41.584682 41.368504 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.367188 67.460938 67.335938 +util 95.750000 100.000000 89.000000 +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 291 : avg iteration time 44.72 seconds +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 291 : data loading time 0.00 +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 291 : avg forward pass time 11.46 +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 291 : backward pass time 10.14 +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 291 : optimizer step time 1.56 +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 291-------------------- +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:54:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441136 41.584717 41.368504 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.430664 67.714844 67.335938 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 292 : avg iteration time 48.60 seconds +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 292 : data loading time 0.00 +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 292 : avg forward pass time 13.38 +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 292 : backward pass time 10.14 +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 292 : optimizer step time 1.55 +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 292-------------------- +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:55:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441137 41.584694 41.368511 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.500000 67.316406 +util 92.500000 99.000000 88.000000 +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 293 : avg iteration time 46.06 seconds +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 293 : data loading time 0.00 +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 293 : avg forward pass time 12.10 +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 293 : backward pass time 10.13 +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 293 : optimizer step time 1.57 +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 293-------------------- +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:56:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441453 41.585953 41.368504 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.464844 67.792969 67.355469 +util 94.000000 99.000000 87.000000 +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 294 : avg iteration time 46.11 seconds +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 294 : data loading time 0.00 +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 294 : avg forward pass time 12.13 +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 294 : backward pass time 10.15 +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 294 : optimizer step time 1.57 +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 294-------------------- +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:56:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441216 41.584999 41.368511 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.570312 107.564453 +util 64.750000 72.000000 56.000000 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 295 : avg iteration time 163.68 seconds +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 295 : data loading time 0.00 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 295 : avg forward pass time 73.20 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 295 : backward pass time 8.43 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 295 : optimizer step time 0.40 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 295-------------------- +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6104 iter count: 1.0 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6104 iter count: 1.0 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6104 iter count: 1.0 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6104iter count: 1.0 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 00:59:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441219 41.585003 41.368511 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.421875 67.396484 +util 93.500000 99.000000 89.000000 +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 296 : avg iteration time 44.47 seconds +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 296 : data loading time 0.00 +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 296 : avg forward pass time 11.32 +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 296 : backward pass time 10.12 +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 296 : optimizer step time 1.56 +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 296-------------------- +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:00:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441282 41.585255 41.368511 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.534668 67.656250 67.494141 +util 93.250000 99.000000 90.000000 +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 297 : avg iteration time 43.34 seconds +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 297 : data loading time 0.00 +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 297 : avg forward pass time 10.75 +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 297 : backward pass time 10.14 +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 297 : optimizer step time 1.57 +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 297-------------------- +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:00:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441292 41.585293 41.368511 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.617188 67.337891 +util 97.750000 100.000000 92.000000 +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 298 : avg iteration time 48.77 seconds +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 298 : data loading time 0.00 +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 298 : avg forward pass time 13.45 +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 298 : backward pass time 10.15 +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 298 : optimizer step time 1.57 +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 298-------------------- +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441309 41.585255 41.368523 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.397949 67.871094 67.240234 +util 93.000000 99.000000 88.000000 +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 299 : avg iteration time 46.02 seconds +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 299 : data loading time 0.00 +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 299 : avg forward pass time 12.10 +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 299 : backward pass time 10.13 +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 299 : optimizer step time 1.56 +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 299-------------------- +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441308 41.585255 41.368534 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.687500 107.583984 +util 62.250000 67.000000 55.000000 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 300 : avg iteration time 161.11 seconds +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 300 : data loading time 0.00 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 300 : avg forward pass time 71.96 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 300 : backward pass time 8.38 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 300 : optimizer step time 0.39 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 300-------------------- +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6504 iter count: 1.0 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6504 iter count: 1.0 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6504 iter count: 1.0 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6504iter count: 1.0 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 01:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 7, 2026 - 01:05:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.36 seconds +[May 7, 2026 - 01:05:28 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 300 +[May 7, 2026 - 01:05:28 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000300 +[May 7, 2026 - 01:05:47 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 300 +[May 7, 2026 - 01:05:47 | INFO | fastgen.trainer:validate:358 ] Validation iteration 300 +[May 7, 2026 - 01:05:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:05:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 7, 2026 - 01:05:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.122 +[May 7, 2026 - 01:05:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:05:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:05:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.172 +[May 7, 2026 - 01:05:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:06:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:06:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.556 +[May 7, 2026 - 01:06:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:06:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:06:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.568 +[May 7, 2026 - 01:06:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:06:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:06:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.760 +[May 7, 2026 - 01:06:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:06:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:06:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.981 +[May 7, 2026 - 01:06:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:06:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:06:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=7.030 +[May 7, 2026 - 01:06:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:06:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:06:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.025 +[May 7, 2026 - 01:06:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:06:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:06:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.672 +[May 7, 2026 - 01:06:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 01:06:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 01:06:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.305 +[May 7, 2026 - 01:06:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 300-------------------- +[May 7, 2026 - 01:06:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 7, 2026 - 01:06:54 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 300 +[May 7, 2026 - 01:06:54 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.519 (n=10) +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.443069 41.591129 41.368958 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.231445 68.281250 68.214844 +util 96.250000 99.000000 90.000000 +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 301 : avg iteration time 145.44 seconds +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 301 : data loading time 0.00 +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 301 : avg forward pass time 11.77 +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 301 : backward pass time 10.13 +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 301 : optimizer step time 1.57 +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 301-------------------- +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:07:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.443069 41.591129 41.368958 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.265625 68.273438 68.242188 +util 94.000000 100.000000 89.000000 +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 302 : avg iteration time 46.26 seconds +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 302 : data loading time 0.00 +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 302 : avg forward pass time 12.17 +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 302 : backward pass time 10.18 +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 302 : optimizer step time 1.57 +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 302-------------------- +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442802 41.590023 41.368973 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.280273 68.300781 68.273438 +util 92.250000 98.000000 87.000000 +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 303 : avg iteration time 46.23 seconds +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 303 : data loading time 0.00 +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 303 : avg forward pass time 12.18 +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 303 : backward pass time 10.15 +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 303 : optimizer step time 1.58 +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 303-------------------- +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442797 41.590023 41.368965 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.255859 68.320312 68.234375 +util 95.750000 99.000000 89.000000 +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 304 : avg iteration time 48.90 seconds +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 304 : data loading time 0.00 +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 304 : avg forward pass time 13.49 +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 304 : backward pass time 10.15 +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 304 : optimizer step time 1.58 +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 304-------------------- +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:10:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441865 41.586239 41.368988 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.628906 107.525391 +util 61.500000 66.000000 57.000000 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 305 : avg iteration time 166.11 seconds +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 305 : data loading time 0.00 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 305 : avg forward pass time 74.43 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 305 : backward pass time 8.44 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 305 : optimizer step time 0.40 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 305-------------------- +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7080 iter count: 1.0 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7080 iter count: 1.0 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7080 iter count: 1.0 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7080iter count: 1.0 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:12:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441854 41.586216 41.368965 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.582031 67.628906 67.441406 +util 96.250000 99.000000 90.000000 +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 306 : avg iteration time 44.91 seconds +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 306 : data loading time 0.00 +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 306 : avg forward pass time 11.53 +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 306 : backward pass time 10.15 +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 306 : optimizer step time 1.57 +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 306-------------------- +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:13:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441917 41.586464 41.368965 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.606445 67.667969 67.421875 +util 93.750000 99.000000 89.000000 +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 307 : avg iteration time 46.07 seconds +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 307 : data loading time 0.00 +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 307 : avg forward pass time 12.10 +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 307 : backward pass time 10.15 +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 307 : optimizer step time 1.58 +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 307-------------------- +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442126 41.587292 41.368973 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.704102 67.746094 67.578125 +util 96.250000 99.000000 90.000000 +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 308 : avg iteration time 46.11 seconds +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 308 : data loading time 0.00 +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 308 : avg forward pass time 12.13 +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 308 : backward pass time 10.13 +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 308 : optimizer step time 1.57 +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 308-------------------- +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:15:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442129 41.587292 41.368973 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.582031 67.648438 67.382812 +util 94.250000 99.000000 90.000000 +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 309 : avg iteration time 45.92 seconds +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 309 : data loading time 0.00 +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 309 : avg forward pass time 12.04 +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 309 : backward pass time 10.14 +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 309 : optimizer step time 1.56 +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 309-------------------- +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:15:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441746 41.585739 41.368992 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.697754 107.804688 107.662109 +util 62.750000 68.000000 57.000000 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 310 : avg iteration time 158.42 seconds +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 310 : data loading time 0.00 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 310 : avg forward pass time 70.58 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 310 : backward pass time 8.41 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 310 : optimizer step time 0.42 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 310-------------------- +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6270 iter count: 1.0 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6270 iter count: 1.0 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6270 iter count: 1.0 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6270iter count: 1.0 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:18:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441741 41.585739 41.368973 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.300293 67.318359 67.246094 +util 93.500000 99.000000 89.000000 +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 311 : avg iteration time 47.17 seconds +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 311 : data loading time 0.00 +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 311 : avg forward pass time 12.68 +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 311 : backward pass time 10.12 +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 311 : optimizer step time 1.56 +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 311-------------------- +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:19:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441868 41.586246 41.368973 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.441895 67.871094 67.298828 +util 94.000000 100.000000 89.000000 +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 312 : avg iteration time 46.16 seconds +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 312 : data loading time 0.00 +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 312 : avg forward pass time 12.13 +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 312 : backward pass time 10.14 +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 312 : optimizer step time 1.58 +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 312-------------------- +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:20:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442119 41.587246 41.368973 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.524902 67.792969 67.435547 +util 95.000000 100.000000 88.000000 +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 313 : avg iteration time 46.16 seconds +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 313 : data loading time 0.00 +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 313 : avg forward pass time 12.11 +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 313 : backward pass time 10.14 +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 313 : optimizer step time 1.58 +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 313-------------------- +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:20:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442182 41.587498 41.368973 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.490723 67.890625 67.357422 +util 93.750000 99.000000 86.000000 +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 314 : avg iteration time 43.73 seconds +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 314 : data loading time 0.00 +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 314 : avg forward pass time 10.94 +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 314 : backward pass time 10.14 +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 314 : optimizer step time 1.57 +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 314-------------------- +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:21:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441802 41.585957 41.368996 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.642578 107.589844 +util 65.750000 71.000000 58.000000 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 315 : avg iteration time 160.84 seconds +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 315 : data loading time 0.00 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 315 : avg forward pass time 71.81 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 315 : backward pass time 8.46 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 315 : optimizer step time 0.39 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 315-------------------- +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6221 iter count: 1.0 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6221 iter count: 1.0 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6221 iter count: 1.0 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6221iter count: 1.0 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:24:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441806 41.585964 41.368984 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.363281 67.316406 +util 96.500000 100.000000 90.000000 +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 316 : avg iteration time 47.20 seconds +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 316 : data loading time 0.00 +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 316 : avg forward pass time 12.65 +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 316 : backward pass time 10.14 +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 316 : optimizer step time 1.58 +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 316-------------------- +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441941 41.586456 41.369011 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.656250 67.316406 +util 94.500000 99.000000 90.000000 +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 317 : avg iteration time 43.48 seconds +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 317 : data loading time 0.00 +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 317 : avg forward pass time 10.80 +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 317 : backward pass time 10.18 +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 317 : optimizer step time 1.56 +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 317-------------------- +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:25:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442254 41.587696 41.369034 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.460938 67.355469 +util 95.500000 100.000000 89.000000 +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 318 : avg iteration time 45.97 seconds +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 318 : data loading time 0.00 +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 318 : avg forward pass time 12.06 +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 318 : backward pass time 10.13 +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 318 : optimizer step time 1.57 +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 318-------------------- +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:26:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442251 41.587696 41.369022 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.352539 67.460938 67.316406 +util 97.000000 100.000000 91.000000 +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 319 : avg iteration time 43.37 seconds +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 319 : data loading time 0.00 +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 319 : avg forward pass time 10.76 +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 319 : backward pass time 10.12 +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 319 : optimizer step time 1.58 +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 319-------------------- +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:27:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441751 41.585659 41.369038 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.667969 107.564453 +util 66.250000 73.000000 59.000000 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 320 : avg iteration time 163.44 seconds +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 320 : data loading time 0.00 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 320 : avg forward pass time 73.11 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 320 : backward pass time 8.39 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 320 : optimizer step time 0.40 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 320-------------------- +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6885 iter count: 1.0 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6885 iter count: 1.0 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6885 iter count: 1.0 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6885iter count: 1.0 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:29:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441752 41.585674 41.369026 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373535 67.421875 67.357422 +util 92.500000 98.000000 88.000000 +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 321 : avg iteration time 44.63 seconds +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 321 : data loading time 0.00 +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 321 : avg forward pass time 11.41 +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 321 : backward pass time 10.12 +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 321 : optimizer step time 1.57 +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 321-------------------- +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:30:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441748 41.585659 41.369026 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.349121 67.382812 67.337891 +util 91.750000 98.000000 87.000000 +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 322 : avg iteration time 48.60 seconds +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 322 : data loading time 0.00 +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 322 : avg forward pass time 13.38 +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 322 : backward pass time 10.14 +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 322 : optimizer step time 1.56 +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 322-------------------- +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:31:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442004 41.586658 41.369061 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373535 67.539062 67.318359 +util 93.750000 99.000000 86.000000 +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 323 : avg iteration time 48.67 seconds +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 323 : data loading time 0.00 +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 323 : avg forward pass time 13.41 +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 323 : backward pass time 10.13 +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 323 : optimizer step time 1.59 +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 323-------------------- +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:32:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442008 41.586681 41.369041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.539062 67.357422 +util 95.000000 99.000000 88.000000 +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 324 : avg iteration time 43.62 seconds +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 324 : data loading time 0.00 +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 324 : avg forward pass time 10.89 +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 324 : backward pass time 10.12 +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 324 : optimizer step time 1.58 +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 324-------------------- +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:33:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441375 41.584148 41.369041 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.648438 107.544922 +util 62.500000 67.000000 58.000000 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 325 : avg iteration time 158.14 seconds +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 325 : data loading time 0.00 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 325 : avg forward pass time 70.50 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 325 : backward pass time 8.40 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 325 : optimizer step time 0.40 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 325-------------------- +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5898 iter count: 1.0 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5898 iter count: 1.0 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5898 iter count: 1.0 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5898iter count: 1.0 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:35:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441383 41.584148 41.369064 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.743164 67.804688 67.558594 +util 93.000000 99.000000 88.000000 +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 326 : avg iteration time 47.35 seconds +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 326 : data loading time 0.00 +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 326 : avg forward pass time 12.72 +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 326 : backward pass time 10.20 +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 326 : optimizer step time 1.57 +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 326-------------------- +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:36:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441432 41.584400 41.369041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.843750 67.480469 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 327 : avg iteration time 48.91 seconds +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 327 : data loading time 0.00 +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 327 : avg forward pass time 13.53 +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 327 : backward pass time 10.15 +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 327 : optimizer step time 1.56 +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 327-------------------- +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:37:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441742 41.585640 41.369041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.645508 67.707031 67.460938 +util 93.000000 99.000000 88.000000 +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 328 : avg iteration time 48.70 seconds +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 328 : data loading time 0.00 +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 328 : avg forward pass time 13.38 +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 328 : backward pass time 10.23 +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 328 : optimizer step time 1.57 +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 328-------------------- +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:38:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441742 41.585640 41.369041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.816406 67.882812 67.617188 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 329 : avg iteration time 45.93 seconds +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 329 : data loading time 0.00 +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 329 : avg forward pass time 12.04 +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 329 : backward pass time 10.13 +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 329 : optimizer step time 1.57 +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 329-------------------- +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:38:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441371 41.584145 41.369041 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.628906 107.564453 +util 69.250000 74.000000 62.000000 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 330 : avg iteration time 166.86 seconds +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 330 : data loading time 0.00 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 330 : avg forward pass time 74.81 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 330 : backward pass time 8.43 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 330 : optimizer step time 0.40 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 330-------------------- +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6602 iter count: 1.0 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6602 iter count: 1.0 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6602 iter count: 1.0 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6602iter count: 1.0 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:41:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441371 41.584141 41.369041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412598 67.519531 67.376953 +util 94.750000 99.000000 91.000000 +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 331 : avg iteration time 47.29 seconds +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 331 : data loading time 0.00 +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 331 : avg forward pass time 12.72 +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 331 : backward pass time 10.14 +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 331 : optimizer step time 1.58 +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 331-------------------- +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:42:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441433 41.584393 41.369041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.788574 67.826172 67.675781 +util 95.000000 99.000000 88.000000 +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 332 : avg iteration time 48.65 seconds +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 332 : data loading time 0.00 +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 332 : avg forward pass time 13.39 +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 332 : backward pass time 10.15 +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 332 : optimizer step time 1.56 +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 332-------------------- +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:43:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441810 41.585899 41.369041 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.471191 67.636719 67.416016 +util 96.500000 100.000000 90.000000 +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 333 : avg iteration time 43.43 seconds +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 333 : data loading time 0.00 +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 333 : avg forward pass time 10.75 +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 333 : backward pass time 10.15 +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 333 : optimizer step time 1.57 +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 333-------------------- +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:43:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441878 41.586132 41.369061 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.412598 67.460938 67.396484 +util 96.500000 100.000000 90.000000 +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 334 : avg iteration time 48.81 seconds +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 334 : data loading time 0.00 +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 334 : avg forward pass time 13.47 +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 334 : backward pass time 10.13 +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 334 : optimizer step time 1.58 +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 334-------------------- +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:44:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441352 41.584064 41.369045 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.589844 107.544922 +util 65.000000 70.000000 57.000000 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 335 : avg iteration time 163.80 seconds +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 335 : data loading time 0.00 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 335 : avg forward pass time 73.22 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 335 : backward pass time 8.46 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 335 : optimizer step time 0.41 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 335-------------------- +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6602 iter count: 1.0 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6602 iter count: 1.0 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6602 iter count: 1.0 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6602iter count: 1.0 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:47:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441373 41.584087 41.369061 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.632324 67.708984 67.402344 +util 95.750000 100.000000 89.000000 +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 336 : avg iteration time 47.18 seconds +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 336 : data loading time 0.00 +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 336 : avg forward pass time 12.66 +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 336 : backward pass time 10.16 +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 336 : optimizer step time 1.55 +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 336-------------------- +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:48:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441422 41.584316 41.369045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.681152 67.787109 67.363281 +util 96.750000 99.000000 91.000000 +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 337 : avg iteration time 43.45 seconds +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 337 : data loading time 0.00 +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 337 : avg forward pass time 10.79 +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 337 : backward pass time 10.13 +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 337 : optimizer step time 1.59 +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 337-------------------- +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:49:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441793 41.585815 41.369045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.637207 67.728516 67.363281 +util 96.750000 99.000000 91.000000 +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 338 : avg iteration time 48.58 seconds +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 338 : data loading time 0.00 +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 338 : avg forward pass time 13.37 +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 338 : backward pass time 10.14 +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 338 : optimizer step time 1.57 +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 338-------------------- +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:49:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441799 41.585815 41.369045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.686035 67.787109 67.382812 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 339 : avg iteration time 45.96 seconds +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 339 : data loading time 0.00 +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 339 : avg forward pass time 12.06 +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 339 : backward pass time 10.11 +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 339 : optimizer step time 1.55 +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 339-------------------- +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:50:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441529 41.584717 41.369045 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.956543 108.052734 107.667969 +util 64.500000 70.000000 58.000000 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 340 : avg iteration time 157.97 seconds +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 340 : data loading time 0.00 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 340 : avg forward pass time 70.39 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 340 : backward pass time 8.43 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 340 : optimizer step time 0.40 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 340-------------------- +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6562 iter count: 1.0 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6562 iter count: 1.0 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6562 iter count: 1.0 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6562iter count: 1.0 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441522 41.584694 41.369061 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.705566 67.728516 67.636719 +util 95.000000 99.000000 88.000000 +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 341 : avg iteration time 44.47 seconds +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 341 : data loading time 0.00 +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 341 : avg forward pass time 11.31 +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 341 : backward pass time 10.14 +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 341 : optimizer step time 1.58 +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 341-------------------- +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:53:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441526 41.584694 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.671387 67.689453 67.617188 +util 93.250000 99.000000 88.000000 +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 342 : avg iteration time 43.67 seconds +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 342 : data loading time 0.00 +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 342 : avg forward pass time 10.90 +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 342 : backward pass time 10.14 +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 342 : optimizer step time 1.58 +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 342-------------------- +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:54:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442024 41.586674 41.369068 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.622559 67.689453 67.421875 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 343 : avg iteration time 45.92 seconds +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 343 : data loading time 0.00 +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 343 : avg forward pass time 12.02 +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 343 : backward pass time 10.15 +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 343 : optimizer step time 1.59 +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 343-------------------- +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442092 41.586948 41.369064 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.676270 67.689453 67.636719 +util 92.500000 98.000000 88.000000 +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 344 : avg iteration time 48.70 seconds +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 344 : data loading time 0.00 +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 344 : avg forward pass time 13.42 +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 344 : backward pass time 10.15 +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 344 : optimizer step time 1.58 +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 344-------------------- +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:56:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442080 41.586926 41.369061 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.225098 108.365234 107.804688 +util 62.000000 67.000000 57.000000 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 345 : avg iteration time 157.82 seconds +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 345 : data loading time 0.00 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 345 : avg forward pass time 70.30 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 345 : backward pass time 8.42 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 345 : optimizer step time 0.41 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 345-------------------- +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6084 iter count: 1.0 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6084 iter count: 1.0 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6084 iter count: 1.0 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6084iter count: 1.0 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:58:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442008 41.586613 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.627441 67.689453 67.441406 +util 97.000000 100.000000 91.000000 +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 346 : avg iteration time 49.70 seconds +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 346 : data loading time 0.00 +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 346 : avg forward pass time 13.93 +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 346 : backward pass time 10.14 +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 346 : optimizer step time 1.57 +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 346-------------------- +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 01:59:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442010 41.586597 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.656738 67.689453 67.558594 +util 94.500000 99.000000 90.000000 +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 347 : avg iteration time 46.11 seconds +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 347 : data loading time 0.00 +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 347 : avg forward pass time 12.12 +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 347 : backward pass time 10.15 +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 347 : optimizer step time 1.57 +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 347-------------------- +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:00:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442016 41.586620 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.637207 67.708984 67.421875 +util 94.500000 99.000000 87.000000 +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 348 : avg iteration time 48.65 seconds +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 348 : data loading time 0.00 +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 348 : avg forward pass time 13.40 +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 348 : backward pass time 10.14 +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 348 : optimizer step time 1.56 +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 348-------------------- +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:01:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442017 41.586605 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.642090 67.708984 67.441406 +util 92.000000 97.000000 88.000000 +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 349 : avg iteration time 43.51 seconds +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 349 : data loading time 0.00 +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 349 : avg forward pass time 10.82 +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 349 : backward pass time 10.14 +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 349 : optimizer step time 1.58 +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 349-------------------- +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:02:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442043 41.586620 41.369080 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.623047 107.609375 +util 61.000000 66.000000 57.000000 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 350 : avg iteration time 158.21 seconds +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 350 : data loading time 0.00 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 350 : avg forward pass time 70.50 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 350 : backward pass time 8.38 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 350 : optimizer step time 0.40 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 350-------------------- +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6270 iter count: 1.0 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6270 iter count: 1.0 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6270 iter count: 1.0 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6270iter count: 1.0 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:04:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442009 41.586483 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.720215 67.748047 67.636719 +util 96.000000 100.000000 90.000000 +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 351 : avg iteration time 46.95 seconds +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 351 : data loading time 0.00 +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 351 : avg forward pass time 12.55 +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 351 : backward pass time 10.13 +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 351 : optimizer step time 1.57 +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 351-------------------- +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:05:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442017 41.586514 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729980 67.767578 67.617188 +util 97.000000 100.000000 91.000000 +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 352 : avg iteration time 45.96 seconds +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 352 : data loading time 0.00 +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 352 : avg forward pass time 12.05 +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 352 : backward pass time 10.15 +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 352 : optimizer step time 1.56 +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 352-------------------- +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:06:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442009 41.586483 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.651855 67.728516 67.421875 +util 95.500000 99.000000 92.000000 +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 353 : avg iteration time 48.72 seconds +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 353 : data loading time 0.00 +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 353 : avg forward pass time 13.43 +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 353 : backward pass time 10.14 +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 353 : optimizer step time 1.57 +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 353-------------------- +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:07:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442261 41.587490 41.369080 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.690918 67.728516 67.578125 +util 95.000000 100.000000 89.000000 +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 354 : avg iteration time 43.49 seconds +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 354 : data loading time 0.00 +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 354 : avg forward pass time 10.81 +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 354 : backward pass time 10.14 +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 354 : optimizer step time 1.59 +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 354-------------------- +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:07:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441982 41.586361 41.369087 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.707031 107.564453 +util 66.000000 71.000000 58.000000 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 355 : avg iteration time 160.58 seconds +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 355 : data loading time 0.00 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 355 : avg forward pass time 71.70 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 355 : backward pass time 8.43 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 355 : optimizer step time 0.39 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 355-------------------- +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5957 iter count: 1.0 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5957 iter count: 1.0 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5957 iter count: 1.0 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5957iter count: 1.0 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:10:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441978 41.586349 41.369087 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.591797 67.648438 67.421875 +util 96.250000 99.000000 93.000000 +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 356 : avg iteration time 47.14 seconds +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 356 : data loading time 0.00 +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 356 : avg forward pass time 12.64 +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 356 : backward pass time 10.14 +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 356 : optimizer step time 1.57 +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 356-------------------- +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:11:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441978 41.586349 41.369087 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.630859 67.636719 67.628906 +util 95.250000 99.000000 89.000000 +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 357 : avg iteration time 43.50 seconds +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 357 : data loading time 0.00 +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 357 : avg forward pass time 10.83 +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 357 : backward pass time 10.12 +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 357 : optimizer step time 1.57 +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 357-------------------- +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:11:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441983 41.586372 41.369087 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.804688 67.636719 +util 92.000000 97.000000 88.000000 +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 358 : avg iteration time 43.44 seconds +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 358 : data loading time 0.00 +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 358 : avg forward pass time 10.79 +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 358 : backward pass time 10.14 +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 358 : optimizer step time 1.57 +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 358-------------------- +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:12:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442298 41.587608 41.369110 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.645508 67.707031 67.460938 +util 96.250000 99.000000 93.000000 +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 359 : avg iteration time 43.26 seconds +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 359 : data loading time 0.00 +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 359 : avg forward pass time 10.72 +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 359 : backward pass time 10.12 +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 359 : optimizer step time 1.56 +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 359-------------------- +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:13:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442047 41.586620 41.369095 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.561035 107.667969 107.525391 +util 68.000000 73.000000 60.000000 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 360 : avg iteration time 160.69 seconds +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 360 : data loading time 0.00 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 360 : avg forward pass time 71.76 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 360 : backward pass time 8.41 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 360 : optimizer step time 0.40 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 360-------------------- +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6240 iter count: 1.0 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6240 iter count: 1.0 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6240 iter count: 1.0 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6240iter count: 1.0 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:16:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442050 41.586632 41.369095 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.607910 67.728516 67.246094 +util 95.500000 99.000000 92.000000 +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 361 : avg iteration time 47.10 seconds +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 361 : data loading time 0.00 +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 361 : avg forward pass time 12.62 +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 361 : backward pass time 10.12 +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 361 : optimizer step time 1.57 +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 361-------------------- +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:16:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442116 41.586891 41.369095 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.637207 67.708984 67.421875 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 362 : avg iteration time 43.44 seconds +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 362 : data loading time 0.00 +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 362 : avg forward pass time 10.79 +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 362 : backward pass time 10.14 +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 362 : optimizer step time 1.58 +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 362-------------------- +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:17:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442113 41.586868 41.369106 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.622559 67.728516 67.304688 +util 94.000000 99.000000 87.000000 +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 363 : avg iteration time 45.99 seconds +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 363 : data loading time 0.00 +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 363 : avg forward pass time 12.05 +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 363 : backward pass time 10.17 +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 363 : optimizer step time 1.56 +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 363-------------------- +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:18:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442118 41.586868 41.369106 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.612793 67.728516 67.265625 +util 96.500000 99.000000 90.000000 +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 364 : avg iteration time 43.24 seconds +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 364 : data loading time 0.00 +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 364 : avg forward pass time 10.69 +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 364 : backward pass time 10.14 +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 364 : optimizer step time 1.57 +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 364-------------------- +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:19:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441831 41.585716 41.369106 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.609375 107.603516 +util 66.750000 72.000000 59.000000 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 365 : avg iteration time 160.62 seconds +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 365 : data loading time 0.00 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 365 : avg forward pass time 71.72 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 365 : backward pass time 8.39 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 365 : optimizer step time 0.41 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 365-------------------- +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6592 iter count: 1.0 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6592 iter count: 1.0 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6592 iter count: 1.0 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6592iter count: 1.0 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:21:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441968 41.586262 41.369106 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.601562 67.707031 67.285156 +util 96.250000 99.000000 90.000000 +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 366 : avg iteration time 46.98 seconds +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 366 : data loading time 0.00 +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 366 : avg forward pass time 12.56 +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 366 : backward pass time 10.15 +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 366 : optimizer step time 1.56 +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 366-------------------- +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:22:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442432 41.588116 41.369114 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.660156 67.804688 67.226562 +util 96.250000 100.000000 91.000000 +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 367 : avg iteration time 46.06 seconds +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 367 : data loading time 0.00 +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 367 : avg forward pass time 12.11 +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 367 : backward pass time 10.13 +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 367 : optimizer step time 1.55 +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 367-------------------- +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:23:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442462 41.588230 41.369114 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.596680 67.707031 67.265625 +util 95.750000 100.000000 90.000000 +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 368 : avg iteration time 48.58 seconds +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 368 : data loading time 0.00 +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 368 : avg forward pass time 13.35 +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 368 : backward pass time 10.17 +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 368 : optimizer step time 1.57 +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 368-------------------- +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:24:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442478 41.588257 41.369125 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.586914 67.687500 67.285156 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 369 : avg iteration time 48.51 seconds +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 369 : data loading time 0.00 +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 369 : avg forward pass time 13.33 +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 369 : backward pass time 10.13 +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 369 : optimizer step time 1.57 +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 369-------------------- +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:24:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442169 41.587025 41.369125 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.667969 107.564453 +util 61.250000 67.000000 57.000000 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 370 : avg iteration time 163.10 seconds +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 370 : data loading time 0.00 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 370 : avg forward pass time 72.97 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 370 : backward pass time 8.42 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 370 : optimizer step time 0.39 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 370-------------------- +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:27:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442167 41.587025 41.369137 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.671387 67.787109 67.324219 +util 96.250000 99.000000 93.000000 +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 371 : avg iteration time 44.44 seconds +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 371 : data loading time 0.00 +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 371 : avg forward pass time 11.30 +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 371 : backward pass time 10.13 +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 371 : optimizer step time 1.57 +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 371-------------------- +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:28:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442259 41.587341 41.369156 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.681152 67.708984 67.597656 +util 95.750000 100.000000 89.000000 +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 372 : avg iteration time 43.43 seconds +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 372 : data loading time 0.00 +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 372 : avg forward pass time 10.79 +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 372 : backward pass time 10.14 +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 372 : optimizer step time 1.56 +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 372-------------------- +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:29:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442263 41.587372 41.369137 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.622559 67.708984 67.363281 +util 96.500000 100.000000 90.000000 +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 373 : avg iteration time 48.61 seconds +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 373 : data loading time 0.00 +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 373 : avg forward pass time 13.37 +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 373 : backward pass time 10.15 +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 373 : optimizer step time 1.58 +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 373-------------------- +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:29:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442260 41.587391 41.369141 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.661621 67.708984 67.519531 +util 92.250000 98.000000 88.000000 +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 374 : avg iteration time 46.10 seconds +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 374 : data loading time 0.00 +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 374 : avg forward pass time 12.11 +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 374 : backward pass time 10.14 +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 374 : optimizer step time 1.58 +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 374-------------------- +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:30:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441935 41.586082 41.369144 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.727051 107.740234 107.687500 +util 61.750000 67.000000 54.000000 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 375 : avg iteration time 160.26 seconds +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 375 : data loading time 0.00 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 375 : avg forward pass time 71.52 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 375 : backward pass time 8.41 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 375 : optimizer step time 0.40 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 375-------------------- +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6816 iter count: 1.0 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6816 iter count: 1.0 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6816 iter count: 1.0 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6816iter count: 1.0 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:33:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441940 41.586082 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.651855 67.728516 67.421875 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 376 : avg iteration time 47.00 seconds +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 376 : data loading time 0.00 +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 376 : avg forward pass time 12.57 +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 376 : backward pass time 10.12 +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 376 : optimizer step time 1.57 +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 376-------------------- +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:34:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442041 41.586445 41.369167 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.646973 67.708984 67.460938 +util 94.750000 99.000000 88.000000 +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 377 : avg iteration time 46.00 seconds +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 377 : data loading time 0.00 +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 377 : avg forward pass time 12.08 +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 377 : backward pass time 10.15 +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 377 : optimizer step time 1.56 +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 377-------------------- +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:34:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442224 41.587196 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.661621 67.787109 67.285156 +util 97.250000 99.000000 92.000000 +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 378 : avg iteration time 46.05 seconds +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 378 : data loading time 0.00 +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 378 : avg forward pass time 12.10 +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 378 : backward pass time 10.14 +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 378 : optimizer step time 1.57 +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 378-------------------- +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:35:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442226 41.587196 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.651855 67.728516 67.421875 +util 96.000000 100.000000 90.000000 +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 379 : avg iteration time 46.03 seconds +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 379 : data loading time 0.00 +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 379 : avg forward pass time 12.08 +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 379 : backward pass time 10.13 +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 379 : optimizer step time 1.57 +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 379-------------------- +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:36:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441824 41.585579 41.369152 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.667969 107.603516 +util 62.750000 67.000000 59.000000 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 380 : avg iteration time 160.26 seconds +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 380 : data loading time 0.00 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 380 : avg forward pass time 71.56 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 380 : backward pass time 8.38 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 380 : optimizer step time 0.40 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 380-------------------- +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6270 iter count: 1.0 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6270 iter count: 1.0 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6270 iter count: 1.0 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6270iter count: 1.0 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:39:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441830 41.585579 41.369167 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.597656 67.335938 +util 97.250000 99.000000 92.000000 +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 381 : avg iteration time 47.10 seconds +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 381 : data loading time 0.00 +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 381 : avg forward pass time 12.63 +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 381 : backward pass time 10.13 +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 381 : optimizer step time 1.56 +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 381-------------------- +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:39:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.441857 41.585667 41.369167 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.464844 67.617188 67.414062 +util 94.500000 99.000000 88.000000 +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 382 : avg iteration time 48.60 seconds +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 382 : data loading time 0.00 +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 382 : avg forward pass time 13.37 +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 382 : backward pass time 10.14 +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 382 : optimizer step time 1.56 +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 382-------------------- +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:40:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442153 41.586895 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.333008 67.441406 67.296875 +util 96.250000 99.000000 90.000000 +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 383 : avg iteration time 43.37 seconds +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 383 : data loading time 0.00 +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 383 : avg forward pass time 10.76 +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 383 : backward pass time 10.13 +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 383 : optimizer step time 1.57 +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 383-------------------- +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:41:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442156 41.586903 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.375000 67.324219 +util 95.750000 100.000000 90.000000 +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 384 : avg iteration time 48.65 seconds +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 384 : data loading time 0.00 +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 384 : avg forward pass time 13.40 +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 384 : backward pass time 10.13 +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 384 : optimizer step time 1.58 +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 384-------------------- +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:42:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442249 41.587269 41.369152 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.697754 107.701172 107.687500 +util 68.250000 75.000000 61.000000 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 385 : avg iteration time 160.62 seconds +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 385 : data loading time 0.00 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 385 : avg forward pass time 71.73 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 385 : backward pass time 8.41 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 385 : optimizer step time 0.39 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 385-------------------- +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6133 iter count: 1.0 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6133 iter count: 1.0 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6133 iter count: 1.0 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6133iter count: 1.0 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:44:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442063 41.586487 41.369190 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.791992 67.882812 67.519531 +util 94.000000 99.000000 87.000000 +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 386 : avg iteration time 49.68 seconds +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 386 : data loading time 0.00 +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 386 : avg forward pass time 13.92 +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 386 : backward pass time 10.14 +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 386 : optimizer step time 1.56 +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 386-------------------- +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:45:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442054 41.586494 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.699219 67.785156 67.441406 +util 97.250000 100.000000 91.000000 +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 387 : avg iteration time 46.05 seconds +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 387 : data loading time 0.00 +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 387 : avg forward pass time 12.11 +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 387 : backward pass time 10.13 +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 387 : optimizer step time 1.57 +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 387-------------------- +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:46:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442110 41.586700 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.816406 67.882812 67.617188 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 388 : avg iteration time 45.97 seconds +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 388 : data loading time 0.00 +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 388 : avg forward pass time 12.05 +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 388 : backward pass time 10.14 +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 388 : optimizer step time 1.57 +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 388-------------------- +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:47:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442174 41.586948 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 97.500000 100.000000 92.000000 +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 389 : avg iteration time 48.63 seconds +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 389 : data loading time 0.00 +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 389 : avg forward pass time 13.40 +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 389 : backward pass time 10.13 +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 389 : optimizer step time 1.57 +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 389-------------------- +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:48:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442182 41.586948 41.369167 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.628906 107.525391 +util 62.250000 67.000000 58.000000 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 390 : avg iteration time 165.74 seconds +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 390 : data loading time 0.00 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 390 : avg forward pass time 74.26 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 390 : backward pass time 8.44 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 390 : optimizer step time 0.40 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 390-------------------- +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6523 iter count: 1.0 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6523 iter count: 1.0 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6523 iter count: 1.0 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6523iter count: 1.0 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:50:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442142 41.586823 41.369144 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.308594 67.578125 67.218750 +util 97.000000 100.000000 94.000000 +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 391 : avg iteration time 47.05 seconds +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 391 : data loading time 0.00 +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 391 : avg forward pass time 12.60 +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 391 : backward pass time 10.15 +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 391 : optimizer step time 1.56 +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 391-------------------- +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:51:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442191 41.586983 41.369148 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.352539 67.402344 67.335938 +util 95.250000 100.000000 89.000000 +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 392 : avg iteration time 45.90 seconds +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 392 : data loading time 0.00 +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 392 : avg forward pass time 12.03 +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 392 : backward pass time 10.13 +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 392 : optimizer step time 1.57 +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 392-------------------- +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:52:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442255 41.587227 41.369148 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.430664 67.597656 67.375000 +util 95.500000 100.000000 89.000000 +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 393 : avg iteration time 46.00 seconds +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 393 : data loading time 0.00 +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 393 : avg forward pass time 12.08 +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 393 : backward pass time 10.13 +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 393 : optimizer step time 1.56 +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 393-------------------- +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:53:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442254 41.587219 41.369171 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.367188 67.578125 67.296875 +util 95.250000 100.000000 89.000000 +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 394 : avg iteration time 43.40 seconds +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 394 : data loading time 0.00 +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 394 : avg forward pass time 10.78 +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 394 : backward pass time 10.14 +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 394 : optimizer step time 1.57 +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 394-------------------- +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:53:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442249 41.587219 41.369167 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.687500 107.544922 +util 67.500000 73.000000 60.000000 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 395 : avg iteration time 155.20 seconds +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 395 : data loading time 0.00 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 395 : avg forward pass time 68.99 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 395 : backward pass time 8.46 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 395 : optimizer step time 0.40 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 395-------------------- +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5903 iter count: 1.0 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5903 iter count: 1.0 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5903 iter count: 1.0 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5903iter count: 1.0 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:56:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442247 41.587219 41.369148 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.293945 67.316406 67.226562 +util 91.750000 97.000000 88.000000 +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 396 : avg iteration time 44.46 seconds +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 396 : data loading time 0.00 +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 396 : avg forward pass time 11.31 +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 396 : backward pass time 10.12 +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 396 : optimizer step time 1.56 +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 396-------------------- +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:57:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442259 41.587219 41.369148 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.352539 67.375000 67.285156 +util 97.250000 99.000000 92.000000 +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 397 : avg iteration time 45.94 seconds +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 397 : data loading time 0.00 +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 397 : avg forward pass time 12.03 +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 397 : backward pass time 10.12 +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 397 : optimizer step time 1.58 +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 397-------------------- +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:57:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442254 41.587219 41.369171 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.333008 67.355469 67.265625 +util 91.250000 96.000000 88.000000 +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 398 : avg iteration time 48.64 seconds +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 398 : data loading time 0.00 +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 398 : avg forward pass time 13.39 +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 398 : backward pass time 10.14 +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 398 : optimizer step time 1.57 +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 398-------------------- +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:58:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442280 41.587334 41.369160 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.289062 67.296875 67.265625 +util 96.750000 99.000000 91.000000 +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 399 : avg iteration time 46.13 seconds +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 399 : data loading time 0.00 +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 399 : avg forward pass time 12.13 +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 399 : backward pass time 10.15 +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 399 : optimizer step time 1.57 +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 399-------------------- +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 02:59:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442099 41.586620 41.369148 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.683105 107.687500 107.681641 +util 63.750000 69.000000 56.000000 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 400 : avg iteration time 155.36 seconds +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 400 : data loading time 0.00 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 400 : avg forward pass time 69.07 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 400 : backward pass time 8.42 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 400 : optimizer step time 0.39 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 400-------------------- +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6191 iter count: 1.0 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6191 iter count: 1.0 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6191 iter count: 1.0 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6191iter count: 1.0 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 03:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 7, 2026 - 03:02:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.22 seconds +[May 7, 2026 - 03:02:21 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 400 +[May 7, 2026 - 03:02:21 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000400 +[May 7, 2026 - 03:02:40 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 400 +[May 7, 2026 - 03:02:41 | INFO | fastgen.trainer:validate:358 ] Validation iteration 400 +[May 7, 2026 - 03:02:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:02:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 7, 2026 - 03:02:46 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.663 +[May 7, 2026 - 03:02:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:02:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:02:51 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.361 +[May 7, 2026 - 03:02:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:02:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:02:56 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.665 +[May 7, 2026 - 03:02:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:02:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:03:01 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.320 +[May 7, 2026 - 03:03:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:03:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:03:06 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.480 +[May 7, 2026 - 03:03:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:03:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:03:11 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.922 +[May 7, 2026 - 03:03:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:03:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:03:16 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.484 +[May 7, 2026 - 03:03:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:03:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:03:21 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.658 +[May 7, 2026 - 03:03:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:03:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:03:26 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.158 +[May 7, 2026 - 03:03:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 03:03:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 03:03:31 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.040 +[May 7, 2026 - 03:03:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 400-------------------- +[May 7, 2026 - 03:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 7, 2026 - 03:03:48 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 400 +[May 7, 2026 - 03:03:48 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.575 (n=10) +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.443795 41.592968 41.369259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.319336 68.515625 68.253906 +util 93.750000 99.000000 90.000000 +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 401 : avg iteration time 146.34 seconds +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 401 : data loading time 0.00 +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 401 : avg forward pass time 11.79 +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 401 : backward pass time 10.15 +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 401 : optimizer step time 1.57 +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 401-------------------- +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:04:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.443795 41.592968 41.369259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.319336 68.515625 68.253906 +util 97.000000 99.000000 91.000000 +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 402 : avg iteration time 43.51 seconds +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 402 : data loading time 0.00 +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 402 : avg forward pass time 10.79 +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 402 : backward pass time 10.17 +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 402 : optimizer step time 1.59 +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 402-------------------- +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:05:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.443858 41.593216 41.369267 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.319336 68.515625 68.253906 +util 96.000000 100.000000 90.000000 +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 403 : avg iteration time 46.23 seconds +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 403 : data loading time 0.00 +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 403 : avg forward pass time 12.16 +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 403 : backward pass time 10.16 +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 403 : optimizer step time 1.57 +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 403-------------------- +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:06:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.443611 41.592243 41.369259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.333984 68.515625 68.273438 +util 95.000000 99.000000 88.000000 +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 404 : avg iteration time 46.14 seconds +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 404 : data loading time 0.00 +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 404 : avg forward pass time 12.12 +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 404 : backward pass time 10.16 +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 404 : optimizer step time 1.60 +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 404-------------------- +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:06:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442545 41.587971 41.369259 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.663574 107.667969 107.662109 +util 63.750000 69.000000 60.000000 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 405 : avg iteration time 163.33 seconds +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 405 : data loading time 0.00 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 405 : avg forward pass time 73.07 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 405 : backward pass time 8.42 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 405 : optimizer step time 0.40 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 405-------------------- +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6240 iter count: 1.0 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6240 iter count: 1.0 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6240 iter count: 1.0 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6240iter count: 1.0 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:09:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442566 41.587971 41.369278 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.728516 67.843750 67.382812 +util 96.000000 99.000000 91.000000 +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 406 : avg iteration time 47.13 seconds +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 406 : data loading time 0.00 +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 406 : avg forward pass time 12.62 +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 406 : backward pass time 10.15 +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 406 : optimizer step time 1.58 +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 406-------------------- +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:10:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442739 41.588696 41.369259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.684570 67.765625 67.441406 +util 94.500000 99.000000 88.000000 +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 407 : avg iteration time 45.98 seconds +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 407 : data loading time 0.00 +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 407 : avg forward pass time 12.04 +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 407 : backward pass time 10.15 +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 407 : optimizer step time 1.58 +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 407-------------------- +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:11:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442915 41.589424 41.369259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.611328 67.667969 67.441406 +util 97.250000 100.000000 91.000000 +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 408 : avg iteration time 43.37 seconds +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 408 : data loading time 0.00 +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 408 : avg forward pass time 10.75 +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 408 : backward pass time 10.14 +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 408 : optimizer step time 1.56 +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 408-------------------- +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:11:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.443038 41.589916 41.369259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.611328 67.667969 67.441406 +util 96.500000 100.000000 90.000000 +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 409 : avg iteration time 45.99 seconds +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 409 : data loading time 0.00 +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 409 : avg forward pass time 12.05 +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 409 : backward pass time 10.16 +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 409 : optimizer step time 1.57 +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 409-------------------- +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:12:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442611 41.588200 41.369267 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.603516 107.589844 +util 61.500000 66.000000 58.000000 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 410 : avg iteration time 160.51 seconds +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 410 : data loading time 0.00 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 410 : avg forward pass time 71.65 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 410 : backward pass time 8.39 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 410 : optimizer step time 0.40 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 410-------------------- +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5732 iter count: 1.0 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5732 iter count: 1.0 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5732 iter count: 1.0 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5732iter count: 1.0 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:15:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442616 41.588203 41.369259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.884766 67.890625 67.882812 +util 95.000000 100.000000 88.000000 +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 411 : avg iteration time 49.72 seconds +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 411 : data loading time 0.00 +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 411 : avg forward pass time 13.94 +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 411 : backward pass time 10.13 +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 411 : optimizer step time 1.58 +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 411-------------------- +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442720 41.588619 41.369259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.894531 67.929688 67.882812 +util 95.000000 99.000000 89.000000 +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 412 : avg iteration time 46.02 seconds +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 412 : data loading time 0.00 +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 412 : avg forward pass time 12.07 +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 412 : backward pass time 10.13 +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 412 : optimizer step time 1.58 +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 412-------------------- +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:16:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442903 41.589363 41.369267 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.889648 67.910156 67.882812 +util 94.500000 99.000000 88.000000 +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 413 : avg iteration time 45.93 seconds +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 413 : data loading time 0.00 +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 413 : avg forward pass time 12.05 +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 413 : backward pass time 10.14 +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 413 : optimizer step time 1.57 +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 413-------------------- +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:17:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442912 41.589363 41.369289 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.909180 67.929688 67.902344 +util 97.000000 99.000000 91.000000 +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 414 : avg iteration time 46.04 seconds +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 414 : data loading time 0.00 +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 414 : avg forward pass time 12.09 +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 414 : backward pass time 10.14 +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 414 : optimizer step time 1.58 +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 414-------------------- +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:18:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442915 41.589363 41.369316 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.707031 107.544922 +util 67.250000 73.000000 60.000000 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 415 : avg iteration time 157.85 seconds +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 415 : data loading time 0.00 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 415 : avg forward pass time 70.33 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 415 : backward pass time 8.40 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 415 : optimizer step time 0.40 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 415-------------------- +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6270 iter count: 1.0 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6270 iter count: 1.0 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6270 iter count: 1.0 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6270iter count: 1.0 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:21:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442706 41.588497 41.369316 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.718750 67.843750 67.343750 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 416 : avg iteration time 47.12 seconds +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 416 : data loading time 0.00 +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 416 : avg forward pass time 12.61 +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 416 : backward pass time 10.14 +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 416 : optimizer step time 1.58 +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 416-------------------- +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:21:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442704 41.588497 41.369316 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.826172 67.941406 67.480469 +util 93.750000 99.000000 90.000000 +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 417 : avg iteration time 43.33 seconds +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 417 : data loading time 0.00 +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 417 : avg forward pass time 10.74 +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 417 : backward pass time 10.13 +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 417 : optimizer step time 1.58 +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 417-------------------- +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 03:22:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:22:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442712 41.588524 41.369316 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.772461 67.882812 67.441406 +util 94.500000 99.000000 91.000000 +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 418 : avg iteration time 48.61 seconds +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 418 : data loading time 0.00 +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 418 : avg forward pass time 13.38 +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 418 : backward pass time 10.14 +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 418 : optimizer step time 1.58 +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 418-------------------- +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:23:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442459 41.587536 41.369316 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.884766 67.890625 67.882812 +util 96.250000 100.000000 91.000000 +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 419 : avg iteration time 48.54 seconds +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 419 : data loading time 0.00 +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 419 : avg forward pass time 13.36 +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 419 : backward pass time 10.14 +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 419 : optimizer step time 1.55 +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 419-------------------- +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:24:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442520 41.587780 41.369316 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.663574 107.667969 107.662109 +util 61.250000 66.000000 58.000000 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 420 : avg iteration time 163.23 seconds +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 420 : data loading time 0.00 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 420 : avg forward pass time 73.02 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 420 : backward pass time 8.41 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 420 : optimizer step time 0.40 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 420-------------------- +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:26:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442521 41.587780 41.369316 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.835938 67.863281 67.753906 +util 96.250000 100.000000 91.000000 +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 421 : avg iteration time 49.77 seconds +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 421 : data loading time 0.00 +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 421 : avg forward pass time 13.95 +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 421 : backward pass time 10.15 +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 421 : optimizer step time 1.58 +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 421-------------------- +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:27:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442526 41.587780 41.369320 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.845703 67.902344 67.675781 +util 92.250000 98.000000 89.000000 +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 422 : avg iteration time 43.43 seconds +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 422 : data loading time 0.00 +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 422 : avg forward pass time 10.79 +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 422 : backward pass time 10.13 +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 422 : optimizer step time 1.58 +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 422-------------------- +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:28:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442522 41.587780 41.369320 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.840820 67.882812 67.714844 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 423 : avg iteration time 46.08 seconds +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 423 : data loading time 0.00 +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 423 : avg forward pass time 12.11 +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 423 : backward pass time 10.13 +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 423 : optimizer step time 1.58 +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 423-------------------- +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:29:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442289 41.586819 41.369339 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.831055 67.882812 67.675781 +util 95.250000 99.000000 92.000000 +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 424 : avg iteration time 45.96 seconds +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 424 : data loading time 0.00 +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 424 : avg forward pass time 12.05 +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 424 : backward pass time 10.13 +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 424 : optimizer step time 1.58 +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 424-------------------- +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:29:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442342 41.587063 41.369320 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.712402 107.720703 107.687500 +util 65.500000 71.000000 58.000000 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 425 : avg iteration time 155.27 seconds +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 425 : data loading time 0.00 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 425 : avg forward pass time 69.01 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 425 : backward pass time 8.42 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 425 : optimizer step time 0.40 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 425-------------------- +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5576 iter count: 1.0 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5576 iter count: 1.0 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5576 iter count: 1.0 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5576iter count: 1.0 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:32:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442343 41.587063 41.369320 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.655273 67.910156 67.570312 +util 91.750000 97.000000 88.000000 +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 426 : avg iteration time 49.88 seconds +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 426 : data loading time 0.00 +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 426 : avg forward pass time 14.03 +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 426 : backward pass time 10.13 +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 426 : optimizer step time 1.57 +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 426-------------------- +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:33:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442348 41.587086 41.369320 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.674805 67.929688 67.589844 +util 93.750000 99.000000 86.000000 +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 427 : avg iteration time 48.88 seconds +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 427 : data loading time 0.00 +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 427 : avg forward pass time 13.51 +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 427 : backward pass time 10.14 +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 427 : optimizer step time 1.58 +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 427-------------------- +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:34:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442343 41.587063 41.369320 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.660156 67.929688 67.570312 +util 97.000000 99.000000 91.000000 +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 428 : avg iteration time 46.05 seconds +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 428 : data loading time 0.00 +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 428 : avg forward pass time 12.09 +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 428 : backward pass time 10.14 +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 428 : optimizer step time 1.57 +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 428-------------------- +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:34:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442479 41.587566 41.369339 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.660156 67.929688 67.570312 +util 93.500000 99.000000 90.000000 +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 429 : avg iteration time 46.09 seconds +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 429 : data loading time 0.00 +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 429 : avg forward pass time 12.13 +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 429 : backward pass time 10.14 +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 429 : optimizer step time 1.57 +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 429-------------------- +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:35:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442229 41.586582 41.369324 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.731934 107.740234 107.707031 +util 66.250000 71.000000 63.000000 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 430 : avg iteration time 163.65 seconds +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 430 : data loading time 0.00 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 430 : avg forward pass time 73.20 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 430 : backward pass time 8.40 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 430 : optimizer step time 0.40 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 430-------------------- +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6221 iter count: 1.0 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6221 iter count: 1.0 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6221 iter count: 1.0 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6221iter count: 1.0 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:38:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442227 41.586559 41.369339 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.987305 68.019531 67.890625 +util 95.750000 99.000000 93.000000 +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 431 : avg iteration time 47.19 seconds +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 431 : data loading time 0.00 +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 431 : avg forward pass time 12.66 +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 431 : backward pass time 10.13 +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 431 : optimizer step time 1.58 +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 431-------------------- +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:39:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442231 41.586582 41.369324 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.879883 67.882812 67.871094 +util 94.000000 99.000000 91.000000 +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 432 : avg iteration time 45.97 seconds +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 432 : data loading time 0.00 +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 432 : avg forward pass time 12.07 +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 432 : backward pass time 10.12 +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 432 : optimizer step time 1.56 +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 432-------------------- +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:40:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:40:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442230 41.586559 41.369331 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.879883 67.882812 67.871094 +util 93.750000 99.000000 90.000000 +[May 7, 2026 - 03:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 433 : avg iteration time 43.40 seconds +[May 7, 2026 - 03:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 433 : data loading time 0.00 +[May 7, 2026 - 03:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 433 : avg forward pass time 10.76 +[May 7, 2026 - 03:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 433 : backward pass time 10.13 +[May 7, 2026 - 03:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 433 : optimizer step time 1.57 +[May 7, 2026 - 03:40:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 433-------------------- +[May 7, 2026 - 03:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 03:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 03:40:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:40:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442415 41.587292 41.369350 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 95.500000 100.000000 89.000000 +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 434 : avg iteration time 45.93 seconds +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 434 : data loading time 0.00 +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 434 : avg forward pass time 12.04 +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 434 : backward pass time 10.13 +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 434 : optimizer step time 1.56 +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 434-------------------- +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:41:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442385 41.587170 41.369350 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.623047 107.589844 +util 63.750000 69.000000 60.000000 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 435 : avg iteration time 160.29 seconds +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 435 : data loading time 0.00 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 435 : avg forward pass time 71.57 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 435 : backward pass time 8.40 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 435 : optimizer step time 0.40 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 435-------------------- +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5947 iter count: 1.0 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5947 iter count: 1.0 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5947 iter count: 1.0 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5947iter count: 1.0 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:44:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442380 41.587147 41.369350 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.796875 67.863281 67.597656 +util 96.500000 100.000000 94.000000 +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 436 : avg iteration time 47.02 seconds +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 436 : data loading time 0.00 +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 436 : avg forward pass time 12.59 +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 436 : backward pass time 10.14 +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 436 : optimizer step time 1.57 +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 436-------------------- +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:44:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442441 41.587391 41.369350 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.816406 67.882812 67.617188 +util 95.750000 99.000000 93.000000 +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 437 : avg iteration time 48.60 seconds +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 437 : data loading time 0.00 +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 437 : avg forward pass time 13.37 +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 437 : backward pass time 10.14 +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 437 : optimizer step time 1.58 +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 437-------------------- +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:45:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442442 41.587395 41.369350 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.831055 67.902344 67.617188 +util 97.250000 99.000000 92.000000 +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 438 : avg iteration time 43.48 seconds +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 438 : data loading time 0.00 +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 438 : avg forward pass time 10.80 +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 438 : backward pass time 10.14 +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 438 : optimizer step time 1.58 +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 438-------------------- +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:46:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442443 41.587372 41.369350 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.831055 67.902344 67.617188 +util 96.250000 99.000000 90.000000 +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 439 : avg iteration time 46.58 seconds +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 439 : data loading time 0.00 +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 439 : avg forward pass time 12.35 +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 439 : backward pass time 10.14 +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 439 : optimizer step time 1.57 +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 439-------------------- +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:47:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442332 41.586948 41.369350 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.648926 107.667969 107.642578 +util 66.250000 72.000000 59.000000 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 440 : avg iteration time 165.65 seconds +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 440 : data loading time 0.00 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 440 : avg forward pass time 74.24 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 440 : backward pass time 8.41 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 440 : optimizer step time 0.39 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 440-------------------- +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6367 iter count: 1.0 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6367 iter count: 1.0 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6367 iter count: 1.0 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6367iter count: 1.0 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:50:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442332 41.586948 41.369350 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.489258 67.949219 67.335938 +util 95.000000 99.000000 92.000000 +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 441 : avg iteration time 44.48 seconds +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 441 : data loading time 0.00 +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 441 : avg forward pass time 11.30 +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 441 : backward pass time 10.15 +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 441 : optimizer step time 1.56 +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 441-------------------- +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:50:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442337 41.586948 41.369370 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.323242 67.695312 67.199219 +util 95.750000 100.000000 89.000000 +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 442 : avg iteration time 46.05 seconds +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 442 : data loading time 0.00 +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 442 : avg forward pass time 12.11 +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 442 : backward pass time 10.13 +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 442 : optimizer step time 1.55 +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 442-------------------- +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:51:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442332 41.586948 41.369350 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.821289 67.863281 67.695312 +util 96.000000 100.000000 93.000000 +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 443 : avg iteration time 43.35 seconds +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 443 : data loading time 0.00 +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 443 : avg forward pass time 10.76 +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 443 : backward pass time 10.11 +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 443 : optimizer step time 1.56 +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 443-------------------- +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:52:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442332 41.586948 41.369350 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.695312 67.355469 +util 94.500000 99.000000 87.000000 +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 444 : avg iteration time 46.03 seconds +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 444 : data loading time 0.00 +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 444 : avg forward pass time 12.10 +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 444 : backward pass time 10.13 +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 444 : optimizer step time 1.55 +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 444-------------------- +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:53:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442081 41.585945 41.369350 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.603516 107.589844 +util 64.750000 70.000000 58.000000 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 445 : avg iteration time 163.07 seconds +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 445 : data loading time 0.00 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 445 : avg forward pass time 72.95 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 445 : backward pass time 8.39 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 445 : optimizer step time 0.40 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 445-------------------- +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6230 iter count: 1.0 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6230 iter count: 1.0 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6230 iter count: 1.0 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6230iter count: 1.0 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:55:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442090 41.585957 41.369370 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.882812 67.382812 +util 91.500000 97.000000 88.000000 +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 446 : avg iteration time 49.64 seconds +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 446 : data loading time 0.00 +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 446 : avg forward pass time 13.90 +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 446 : backward pass time 10.13 +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 446 : optimizer step time 1.56 +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 446-------------------- +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:56:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442266 41.586647 41.369385 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.882812 67.421875 +util 95.000000 99.000000 89.000000 +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 447 : avg iteration time 45.95 seconds +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 447 : data loading time 0.00 +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 447 : avg forward pass time 12.05 +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 447 : backward pass time 10.12 +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 447 : optimizer step time 1.58 +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 447-------------------- +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:57:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442262 41.586647 41.369370 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.718750 67.863281 67.285156 +util 91.250000 97.000000 88.000000 +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 448 : avg iteration time 48.45 seconds +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 448 : data loading time 0.00 +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 448 : avg forward pass time 13.32 +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 448 : backward pass time 10.11 +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 448 : optimizer step time 1.57 +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 448-------------------- +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:58:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442262 41.586647 41.369370 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.914062 67.949219 67.902344 +util 94.000000 99.000000 91.000000 +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 449 : avg iteration time 46.03 seconds +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 449 : data loading time 0.00 +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 449 : avg forward pass time 12.06 +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 449 : backward pass time 10.17 +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 449 : optimizer step time 1.57 +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 449-------------------- +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 03:58:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442058 41.585819 41.369377 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.746582 107.765625 107.740234 +util 63.500000 69.000000 56.000000 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 450 : avg iteration time 160.65 seconds +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 450 : data loading time 0.00 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 450 : avg forward pass time 71.74 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 450 : backward pass time 8.40 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 450 : optimizer step time 0.39 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 450-------------------- +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6367 iter count: 1.0 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6367 iter count: 1.0 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6367 iter count: 1.0 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6367iter count: 1.0 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:01:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442059 41.585823 41.369377 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.743164 67.863281 67.382812 +util 96.000000 99.000000 91.000000 +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 451 : avg iteration time 49.70 seconds +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 451 : data loading time 0.00 +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 451 : avg forward pass time 13.91 +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 451 : backward pass time 10.15 +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 451 : optimizer step time 1.57 +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 451-------------------- +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:02:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442241 41.586552 41.369377 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.733398 67.863281 67.343750 +util 95.500000 99.000000 90.000000 +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 452 : avg iteration time 45.99 seconds +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 452 : data loading time 0.00 +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 452 : avg forward pass time 12.05 +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 452 : backward pass time 10.14 +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 452 : optimizer step time 1.58 +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 452-------------------- +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:03:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.442241 41.586552 41.369377 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.733398 67.863281 67.343750 +util 96.000000 99.000000 91.000000 +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 453 : avg iteration time 48.62 seconds +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 453 : data loading time 0.00 +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 453 : avg forward pass time 13.39 +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 453 : backward pass time 10.13 +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 453 : optimizer step time 1.56 +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 453-------------------- +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:03:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.632798 41.785374 41.556095 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 67.882812 67.343750 +util 94.000000 99.000000 91.000000 +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 454 : avg iteration time 48.45 seconds +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 454 : data loading time 0.00 +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 454 : avg forward pass time 12.74 +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 454 : backward pass time 10.13 +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 454 : optimizer step time 1.56 +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 454-------------------- +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:04:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.632657 41.784523 41.556118 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.702637 107.707031 107.701172 +util 64.000000 69.000000 60.000000 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 455 : avg iteration time 158.41 seconds +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 455 : data loading time 0.00 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 455 : avg forward pass time 70.56 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 455 : backward pass time 8.43 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 455 : optimizer step time 0.40 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 455-------------------- +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5977 iter count: 1.0 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5977 iter count: 1.0 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5977 iter count: 1.0 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5977iter count: 1.0 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:07:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.633458 41.785412 41.556362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.441406 67.316406 +util 96.000000 100.000000 89.000000 +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 456 : avg iteration time 47.07 seconds +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 456 : data loading time 0.00 +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 456 : avg forward pass time 12.61 +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 456 : backward pass time 10.14 +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 456 : optimizer step time 1.57 +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 456-------------------- +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:08:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.634109 41.786335 41.556530 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.441406 67.316406 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 457 : avg iteration time 48.59 seconds +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 457 : data loading time 0.00 +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 457 : avg forward pass time 13.37 +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 457 : backward pass time 10.15 +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 457 : optimizer step time 1.58 +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 457-------------------- +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:09:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.634869 41.788441 41.557396 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.245117 67.441406 67.179688 +util 93.750000 99.000000 90.000000 +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 458 : avg iteration time 48.72 seconds +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 458 : data loading time 0.00 +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 458 : avg forward pass time 13.43 +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 458 : backward pass time 10.14 +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 458 : optimizer step time 1.57 +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 458-------------------- +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:09:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.635001 41.788498 41.557598 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.259766 67.441406 67.199219 +util 95.250000 100.000000 88.000000 +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 459 : avg iteration time 48.52 seconds +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 459 : data loading time 0.00 +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 459 : avg forward pass time 13.35 +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 459 : backward pass time 10.12 +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 459 : optimizer step time 1.57 +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 459-------------------- +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:10:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.634867 41.786816 41.557602 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.663574 107.726562 107.642578 +util 65.250000 70.000000 62.000000 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 460 : avg iteration time 160.11 seconds +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 460 : data loading time 0.00 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 460 : avg forward pass time 71.46 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 460 : backward pass time 8.42 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 460 : optimizer step time 0.40 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 460-------------------- +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5674 iter count: 1.0 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5674 iter count: 1.0 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5674 iter count: 1.0 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5674iter count: 1.0 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:13:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.635448 41.786907 41.558594 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.949707 68.099609 67.500000 +util 94.750000 99.000000 88.000000 +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 461 : avg iteration time 44.44 seconds +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 461 : data loading time 0.00 +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 461 : avg forward pass time 11.29 +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 461 : backward pass time 10.12 +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 461 : optimizer step time 1.58 +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 461-------------------- +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:14:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.635901 41.787376 41.558632 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.925293 68.099609 67.402344 +util 92.500000 98.000000 89.000000 +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 462 : avg iteration time 43.32 seconds +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 462 : data loading time 0.00 +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 462 : avg forward pass time 10.73 +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 462 : backward pass time 10.13 +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 462 : optimizer step time 1.57 +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 462-------------------- +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:14:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636832 41.789860 41.558655 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.939941 68.099609 67.460938 +util 96.750000 99.000000 94.000000 +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 463 : avg iteration time 46.06 seconds +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 463 : data loading time 0.00 +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 463 : avg forward pass time 12.07 +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 463 : backward pass time 10.18 +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 463 : optimizer step time 1.57 +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 463-------------------- +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:15:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636877 41.789978 41.558655 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.939941 68.099609 67.460938 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 464 : avg iteration time 46.07 seconds +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 464 : data loading time 0.00 +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 464 : avg forward pass time 12.11 +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 464 : backward pass time 10.15 +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 464 : optimizer step time 1.56 +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 464-------------------- +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:16:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636417 41.787792 41.558655 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.648926 107.662109 107.609375 +util 67.250000 73.000000 60.000000 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 465 : avg iteration time 160.20 seconds +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 465 : data loading time 0.00 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 465 : avg forward pass time 71.51 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 465 : backward pass time 8.41 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 465 : optimizer step time 0.40 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 465-------------------- +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5376 iter count: 1.0 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5376 iter count: 1.0 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5376 iter count: 1.0 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5376iter count: 1.0 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:18:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636845 41.787876 41.559742 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.882812 67.382812 +util 91.250000 97.000000 87.000000 +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 466 : avg iteration time 44.66 seconds +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 466 : data loading time 0.00 +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 466 : avg forward pass time 11.41 +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 466 : backward pass time 10.15 +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 466 : optimizer step time 1.56 +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 466-------------------- +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:19:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.636958 41.788116 41.559837 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.679688 67.824219 67.246094 +util 94.500000 99.000000 91.000000 +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 467 : avg iteration time 48.64 seconds +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 467 : data loading time 0.00 +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 467 : avg forward pass time 13.39 +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 467 : backward pass time 10.15 +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 467 : optimizer step time 1.56 +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 467-------------------- +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:20:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637456 41.788982 41.560818 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.863281 67.421875 +util 93.750000 100.000000 87.000000 +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 468 : avg iteration time 48.75 seconds +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 468 : data loading time 0.00 +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 468 : avg forward pass time 13.44 +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 468 : backward pass time 10.15 +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 468 : optimizer step time 1.57 +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 468-------------------- +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637564 41.789249 41.560886 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.421875 67.355469 +util 92.750000 98.000000 89.000000 +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 469 : avg iteration time 46.02 seconds +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 469 : data loading time 0.00 +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 469 : avg forward pass time 12.08 +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 469 : backward pass time 10.15 +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 469 : optimizer step time 1.57 +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 469-------------------- +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637826 41.789227 41.560925 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.642578 107.609375 +util 68.250000 74.000000 60.000000 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 470 : avg iteration time 155.34 seconds +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 470 : data loading time 0.00 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 470 : avg forward pass time 69.06 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 470 : backward pass time 8.41 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 470 : optimizer step time 0.39 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 470-------------------- +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5918 iter count: 1.0 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5918 iter count: 1.0 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5918 iter count: 1.0 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5918iter count: 1.0 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:24:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.637900 41.789040 41.560890 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.280762 67.318359 67.167969 +util 93.750000 99.000000 90.000000 +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 471 : avg iteration time 44.49 seconds +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 471 : data loading time 0.00 +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 471 : avg forward pass time 11.32 +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 471 : backward pass time 10.14 +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 471 : optimizer step time 1.57 +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 471-------------------- +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:25:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638001 41.789017 41.560890 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.305176 67.337891 67.207031 +util 97.000000 99.000000 91.000000 +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 472 : avg iteration time 48.70 seconds +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 472 : data loading time 0.00 +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 472 : avg forward pass time 13.46 +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 472 : backward pass time 10.10 +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 472 : optimizer step time 1.56 +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 472-------------------- +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:26:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638290 41.789810 41.561127 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.422363 67.494141 67.207031 +util 93.500000 99.000000 87.000000 +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 473 : avg iteration time 46.06 seconds +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 473 : data loading time 0.00 +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 473 : avg forward pass time 12.10 +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 473 : backward pass time 10.15 +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 473 : optimizer step time 1.56 +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 473-------------------- +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:27:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638400 41.789810 41.561230 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.197754 67.201172 67.187500 +util 94.000000 100.000000 88.000000 +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 474 : avg iteration time 48.80 seconds +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 474 : data loading time 0.00 +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 474 : avg forward pass time 13.49 +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 474 : backward pass time 10.15 +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 474 : optimizer step time 1.55 +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 474-------------------- +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:27:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638656 41.790829 41.561234 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.512207 107.589844 107.486328 +util 64.750000 70.000000 57.000000 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 475 : avg iteration time 155.52 seconds +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 475 : data loading time 0.00 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 475 : avg forward pass time 69.16 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 475 : backward pass time 8.43 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 475 : optimizer step time 0.39 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 475-------------------- +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5723 iter count: 1.0 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5723 iter count: 1.0 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5723 iter count: 1.0 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5723iter count: 1.0 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:30:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638866 41.791336 41.561359 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.290527 67.910156 67.083984 +util 96.000000 99.000000 93.000000 +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 476 : avg iteration time 47.16 seconds +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 476 : data loading time 0.00 +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 476 : avg forward pass time 12.65 +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 476 : backward pass time 10.13 +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 476 : optimizer step time 1.58 +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 476-------------------- +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:31:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.638994 41.791710 41.561367 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.510254 67.533203 67.441406 +util 95.750000 100.000000 90.000000 +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 477 : avg iteration time 45.91 seconds +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 477 : data loading time 0.00 +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 477 : avg forward pass time 12.02 +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 477 : backward pass time 10.14 +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 477 : optimizer step time 1.58 +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 477-------------------- +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:31:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639088 41.792126 41.561348 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.910156 67.298828 +util 95.500000 99.000000 90.000000 +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 478 : avg iteration time 46.00 seconds +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 478 : data loading time 0.00 +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 478 : avg forward pass time 12.06 +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 478 : backward pass time 10.14 +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 478 : optimizer step time 1.60 +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 478-------------------- +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:32:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639238 41.792606 41.561340 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.720215 67.871094 67.669922 +util 92.000000 98.000000 88.000000 +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 479 : avg iteration time 48.64 seconds +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 479 : data loading time 0.00 +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 479 : avg forward pass time 13.38 +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 479 : backward pass time 10.15 +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 479 : optimizer step time 1.57 +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 479-------------------- +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:33:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639074 41.791676 41.561340 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.712402 107.720703 107.687500 +util 67.250000 73.000000 60.000000 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 480 : avg iteration time 160.33 seconds +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 480 : data loading time 0.00 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 480 : avg forward pass time 71.54 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 480 : backward pass time 8.47 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 480 : optimizer step time 0.40 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 480-------------------- +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5449 iter count: 1.0 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5449 iter count: 1.0 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5449 iter count: 1.0 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5449iter count: 1.0 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:36:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639084 41.791698 41.561352 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.363770 67.441406 67.337891 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 481 : avg iteration time 46.96 seconds +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 481 : data loading time 0.00 +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 481 : avg forward pass time 12.56 +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 481 : backward pass time 10.12 +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 481 : optimizer step time 1.57 +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 481-------------------- +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:37:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639215 41.791992 41.561390 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.329590 67.421875 67.298828 +util 95.000000 99.000000 88.000000 +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 482 : avg iteration time 48.61 seconds +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 482 : data loading time 0.00 +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 482 : avg forward pass time 13.37 +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 482 : backward pass time 10.15 +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 482 : optimizer step time 1.58 +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 482-------------------- +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:37:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639246 41.792015 41.561378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.344238 67.421875 67.318359 +util 94.750000 99.000000 91.000000 +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 483 : avg iteration time 48.54 seconds +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 483 : data loading time 0.00 +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 483 : avg forward pass time 13.34 +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 483 : backward pass time 10.13 +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 483 : optimizer step time 1.57 +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 483-------------------- +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:38:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639611 41.793213 41.561428 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.139160 67.421875 67.044922 +util 94.500000 99.000000 88.000000 +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 484 : avg iteration time 43.30 seconds +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 484 : data loading time 0.00 +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 484 : avg forward pass time 10.72 +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 484 : backward pass time 10.15 +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 484 : optimizer step time 1.56 +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 484-------------------- +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:39:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639391 41.791870 41.561600 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.623047 107.609375 +util 66.250000 70.000000 63.000000 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 485 : avg iteration time 160.33 seconds +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 485 : data loading time 0.00 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 485 : avg forward pass time 71.55 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 485 : backward pass time 8.43 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 485 : optimizer step time 0.39 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 485-------------------- +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6250 iter count: 1.0 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6250 iter count: 1.0 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6250 iter count: 1.0 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6250iter count: 1.0 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:42:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639445 41.791870 41.561581 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.343262 67.480469 67.296875 +util 95.250000 100.000000 88.000000 +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 486 : avg iteration time 44.63 seconds +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 486 : data loading time 0.00 +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 486 : avg forward pass time 11.38 +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 486 : backward pass time 10.15 +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 486 : optimizer step time 1.56 +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 486-------------------- +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:42:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639536 41.791901 41.561581 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362793 67.500000 67.316406 +util 96.500000 100.000000 90.000000 +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 487 : avg iteration time 48.62 seconds +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 487 : data loading time 0.00 +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 487 : avg forward pass time 13.38 +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 487 : backward pass time 10.14 +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 487 : optimizer step time 1.57 +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 487-------------------- +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:43:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639567 41.791885 41.561584 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.470215 67.929688 67.316406 +util 96.750000 99.000000 94.000000 +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 488 : avg iteration time 45.95 seconds +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 488 : data loading time 0.00 +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 488 : avg forward pass time 12.05 +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 488 : backward pass time 10.13 +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 488 : optimizer step time 1.55 +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 488-------------------- +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:44:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639691 41.792259 41.561604 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.353027 67.460938 67.316406 +util 94.750000 99.000000 88.000000 +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 489 : avg iteration time 46.03 seconds +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 489 : data loading time 0.00 +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 489 : avg forward pass time 12.09 +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 489 : backward pass time 10.14 +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 489 : optimizer step time 1.56 +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 489-------------------- +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:45:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639665 41.792183 41.561588 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.648438 107.603516 +util 63.500000 68.000000 60.000000 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 490 : avg iteration time 163.15 seconds +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 490 : data loading time 0.00 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 490 : avg forward pass time 72.98 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 490 : backward pass time 8.42 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 490 : optimizer step time 0.39 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 490-------------------- +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5986 iter count: 1.0 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5986 iter count: 1.0 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5986 iter count: 1.0 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5986iter count: 1.0 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:47:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639755 41.792206 41.561672 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.324219 67.376953 67.167969 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 491 : avg iteration time 47.08 seconds +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 491 : data loading time 0.00 +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 491 : avg forward pass time 12.62 +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 491 : backward pass time 10.13 +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 491 : optimizer step time 1.57 +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 491-------------------- +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:48:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639877 41.792229 41.561840 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.329102 67.376953 67.187500 +util 94.000000 99.000000 88.000000 +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 492 : avg iteration time 46.16 seconds +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 492 : data loading time 0.00 +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 492 : avg forward pass time 12.15 +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 492 : backward pass time 10.15 +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 492 : optimizer step time 1.57 +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 492-------------------- +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:49:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639932 41.792248 41.561840 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.324219 67.376953 67.167969 +util 94.250000 99.000000 88.000000 +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 493 : avg iteration time 45.92 seconds +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 493 : data loading time 0.00 +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 493 : avg forward pass time 12.04 +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 493 : backward pass time 10.14 +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 493 : optimizer step time 1.58 +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 493-------------------- +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:50:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639956 41.792248 41.561840 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.329102 67.376953 67.187500 +util 95.750000 100.000000 89.000000 +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 494 : avg iteration time 45.91 seconds +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 494 : data loading time 0.00 +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 494 : avg forward pass time 12.03 +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 494 : backward pass time 10.14 +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 494 : optimizer step time 1.57 +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 494-------------------- +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:50:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.639877 41.791920 41.561840 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.642578 107.609375 +util 64.250000 73.000000 57.000000 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 495 : avg iteration time 160.69 seconds +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 495 : data loading time 0.00 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 495 : avg forward pass time 71.74 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 495 : backward pass time 8.41 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 495 : optimizer step time 0.39 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 495-------------------- +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6504 iter count: 1.0 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6504 iter count: 1.0 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6504 iter count: 1.0 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6504iter count: 1.0 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:53:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640089 41.792217 41.561840 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.056152 67.382812 66.947266 +util 96.750000 99.000000 91.000000 +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 496 : avg iteration time 47.14 seconds +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 496 : data loading time 0.00 +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 496 : avg forward pass time 12.63 +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 496 : backward pass time 10.14 +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 496 : optimizer step time 1.57 +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 496-------------------- +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:54:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640241 41.792805 41.561863 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.026855 67.265625 66.947266 +util 94.500000 99.000000 89.000000 +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 497 : avg iteration time 43.28 seconds +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 497 : data loading time 0.00 +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 497 : avg forward pass time 10.72 +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 497 : backward pass time 10.13 +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 497 : optimizer step time 1.57 +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 497-------------------- +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:55:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640291 41.792908 41.561939 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.075684 67.460938 66.947266 +util 94.250000 99.000000 90.000000 +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 498 : avg iteration time 46.03 seconds +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 498 : data loading time 0.00 +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 498 : avg forward pass time 12.09 +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 498 : backward pass time 10.14 +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 498 : optimizer step time 1.56 +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 498-------------------- +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:55:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640353 41.792950 41.561989 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.070801 67.441406 66.947266 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 499 : avg iteration time 46.05 seconds +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 499 : data loading time 0.00 +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 499 : avg forward pass time 12.12 +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 499 : backward pass time 10.13 +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 499 : optimizer step time 1.57 +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 499-------------------- +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:56:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640147 41.792068 41.562050 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.628906 107.583984 +util 64.000000 70.000000 56.000000 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 500 : avg iteration time 160.60 seconds +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 500 : data loading time 0.00 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 500 : avg forward pass time 71.71 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 500 : backward pass time 8.40 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 500 : optimizer step time 0.39 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 500-------------------- +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6133 iter count: 1.0 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6133 iter count: 1.0 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6133 iter count: 1.0 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6133iter count: 1.0 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 04:59:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 7, 2026 - 04:59:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.36 seconds +[May 7, 2026 - 04:59:33 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 500 +[May 7, 2026 - 04:59:33 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000500 +[May 7, 2026 - 04:59:52 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 500 +[May 7, 2026 - 04:59:53 | INFO | fastgen.trainer:validate:358 ] Validation iteration 500 +[May 7, 2026 - 04:59:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 04:59:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 7, 2026 - 04:59:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.170 +[May 7, 2026 - 04:59:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.453 +[May 7, 2026 - 05:00:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.887 +[May 7, 2026 - 05:00:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.779 +[May 7, 2026 - 05:00:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.890 +[May 7, 2026 - 05:00:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.598 +[May 7, 2026 - 05:00:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=7.252 +[May 7, 2026 - 05:00:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.333 +[May 7, 2026 - 05:00:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.674 +[May 7, 2026 - 05:00:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 05:00:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 05:00:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.960 +[May 7, 2026 - 05:00:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 500-------------------- +[May 7, 2026 - 05:00:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 7, 2026 - 05:01:00 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 500 +[May 7, 2026 - 05:01:00 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.400 (n=10) +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642004 41.798817 41.562252 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.392578 68.574219 68.332031 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 501 : avg iteration time 148.40 seconds +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 0.00 +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 13.02 +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 10.16 +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 1.56 +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 501-------------------- +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:01:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642016 41.798817 41.562252 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.377930 68.574219 68.312500 +util 94.250000 99.000000 90.000000 +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 46.32 seconds +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 12.19 +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 10.19 +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 1.57 +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 502-------------------- +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:02:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641641 41.797096 41.562256 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.387695 68.613281 68.312500 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 46.25 seconds +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 12.15 +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 10.19 +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 1.58 +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 503-------------------- +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:03:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641719 41.797127 41.562271 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.280273 68.593750 68.175781 +util 95.750000 100.000000 90.000000 +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 48.68 seconds +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 13.40 +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 10.15 +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 1.57 +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 504-------------------- +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:04:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640580 41.792625 41.562252 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.824219 107.583984 +util 65.500000 72.000000 58.000000 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 160.54 seconds +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 71.67 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 8.40 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.39 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 505-------------------- +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5488 iter count: 1.0 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5488 iter count: 1.0 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5488 iter count: 1.0 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5488iter count: 1.0 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:06:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640602 41.792629 41.562317 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.602539 67.728516 67.226562 +util 92.500000 98.000000 88.000000 +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 47.10 seconds +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 12.62 +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 10.15 +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 1.56 +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 506-------------------- +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:07:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640763 41.793091 41.562325 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.680664 67.767578 67.421875 +util 92.500000 98.000000 88.000000 +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 46.00 seconds +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 12.07 +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 10.14 +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 1.58 +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 507-------------------- +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:08:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641084 41.794327 41.562370 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.641602 67.728516 67.382812 +util 96.250000 100.000000 91.000000 +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 46.16 seconds +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 12.14 +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 10.14 +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 1.57 +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 508-------------------- +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:09:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641090 41.794327 41.562347 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.670898 67.767578 67.382812 +util 95.000000 100.000000 88.000000 +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 48.72 seconds +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 13.42 +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 10.14 +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 1.58 +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 509-------------------- +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640737 41.792728 41.562531 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.667969 107.623047 +util 64.750000 70.000000 58.000000 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 157.86 seconds +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 70.32 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 8.40 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.39 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 510-------------------- +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5869 iter count: 1.0 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5869 iter count: 1.0 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5869 iter count: 1.0 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5869iter count: 1.0 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:12:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640759 41.792839 41.562508 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.646484 67.787109 67.226562 +util 96.000000 100.000000 90.000000 +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 49.60 seconds +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 13.89 +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 10.12 +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 1.57 +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 511-------------------- +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:13:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640928 41.793358 41.562626 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.178711 68.314453 67.773438 +util 94.500000 99.000000 89.000000 +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 46.04 seconds +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 12.08 +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 10.16 +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 1.58 +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 512-------------------- +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:14:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641334 41.794842 41.562641 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.188477 68.333984 67.753906 +util 94.000000 99.000000 87.000000 +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 43.49 seconds +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 10.81 +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 10.16 +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 1.57 +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 513-------------------- +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:14:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641422 41.795105 41.562645 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.188477 68.333984 67.753906 +util 94.000000 99.000000 88.000000 +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 43.31 seconds +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 10.74 +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 10.13 +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 1.57 +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 514-------------------- +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:15:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641018 41.793259 41.562599 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.583984 107.570312 +util 65.000000 70.000000 58.000000 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 160.51 seconds +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 71.68 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 8.38 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.40 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 515-------------------- +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6426 iter count: 1.0 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6426 iter count: 1.0 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6426 iter count: 1.0 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6426iter count: 1.0 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:18:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640838 41.792385 41.562599 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.705566 67.865234 67.226562 +util 95.750000 100.000000 90.000000 +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 49.66 seconds +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 13.91 +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 10.14 +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 1.56 +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 516-------------------- +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:19:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640928 41.792740 41.562599 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.749512 67.865234 67.402344 +util 94.750000 99.000000 88.000000 +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 43.33 seconds +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 10.73 +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 10.14 +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 1.58 +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 517-------------------- +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:19:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641261 41.794044 41.562622 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.739746 67.845703 67.421875 +util 95.750000 99.000000 92.000000 +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 43.36 seconds +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 10.77 +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 10.13 +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 1.56 +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 518-------------------- +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:20:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641392 41.794510 41.562645 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.754395 67.865234 67.421875 +util 94.500000 99.000000 89.000000 +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 45.94 seconds +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 12.04 +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 10.15 +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 1.57 +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 519-------------------- +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:21:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640884 41.792473 41.562649 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.707031 107.603516 +util 63.250000 68.000000 59.000000 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 165.64 seconds +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 74.22 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 8.41 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.40 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 520-------------------- +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5947 iter count: 1.0 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5947 iter count: 1.0 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5947 iter count: 1.0 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5947iter count: 1.0 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:24:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640882 41.792450 41.562668 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.689941 67.845703 67.226562 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 47.07 seconds +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 12.60 +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 10.15 +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 1.56 +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 521-------------------- +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:24:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.640962 41.792732 41.562679 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.665527 67.826172 67.187500 +util 94.750000 99.000000 88.000000 +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 48.59 seconds +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.00 +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 13.37 +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 10.15 +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 1.57 +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 522-------------------- +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:25:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641348 41.794235 41.562656 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.665527 67.826172 67.187500 +util 96.750000 99.000000 91.000000 +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 43.43 seconds +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 10.78 +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 10.14 +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 1.58 +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 523-------------------- +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:26:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:27:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641361 41.794239 41.562656 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.665527 67.826172 67.187500 +util 94.250000 99.000000 90.000000 +[May 7, 2026 - 05:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 524 : avg iteration time 48.50 seconds +[May 7, 2026 - 05:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 524 : data loading time 0.00 +[May 7, 2026 - 05:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 524 : avg forward pass time 13.32 +[May 7, 2026 - 05:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 524 : backward pass time 10.15 +[May 7, 2026 - 05:27:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 524 : optimizer step time 1.57 +[May 7, 2026 - 05:27:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 524-------------------- +[May 7, 2026 - 05:27:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 05:27:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 05:27:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:27:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641024 41.792713 41.562664 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.546387 107.667969 107.505859 +util 66.000000 72.000000 58.000000 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 525 : avg iteration time 157.88 seconds +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 525 : data loading time 0.00 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 525 : avg forward pass time 70.36 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 525 : backward pass time 8.41 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 525 : optimizer step time 0.39 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 525-------------------- +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5396 iter count: 1.0 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5396 iter count: 1.0 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5396 iter count: 1.0 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5396iter count: 1.0 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:29:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641084 41.792820 41.562778 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.680176 67.787109 67.363281 +util 93.750000 99.000000 88.000000 +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 526 : avg iteration time 44.47 seconds +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 526 : data loading time 0.00 +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 526 : avg forward pass time 11.32 +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 526 : backward pass time 10.15 +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 526 : optimizer step time 1.56 +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 526-------------------- +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:30:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641129 41.792820 41.562775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.163574 68.412109 67.421875 +util 94.000000 99.000000 87.000000 +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 527 : avg iteration time 48.49 seconds +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 527 : data loading time 0.00 +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 527 : avg forward pass time 13.34 +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 527 : backward pass time 10.11 +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 527 : optimizer step time 1.57 +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 527-------------------- +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:31:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641327 41.793594 41.562775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.163574 68.412109 67.421875 +util 92.500000 98.000000 88.000000 +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 528 : avg iteration time 43.33 seconds +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 528 : data loading time 0.00 +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 528 : avg forward pass time 10.75 +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 528 : backward pass time 10.11 +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 528 : optimizer step time 1.57 +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 528-------------------- +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:32:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641387 41.793835 41.562775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.163574 68.412109 67.421875 +util 96.250000 99.000000 90.000000 +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 529 : avg iteration time 48.71 seconds +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 529 : data loading time 0.00 +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 529 : avg forward pass time 13.46 +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 529 : backward pass time 10.12 +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 529 : optimizer step time 1.55 +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 529-------------------- +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:32:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641467 41.794144 41.562778 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.673340 107.681641 107.648438 +util 64.250000 70.000000 57.000000 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 530 : avg iteration time 157.79 seconds +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 530 : data loading time 0.00 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 530 : avg forward pass time 70.26 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 530 : backward pass time 8.48 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 530 : optimizer step time 0.39 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 530-------------------- +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6279 iter count: 1.0 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6279 iter count: 1.0 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6279 iter count: 1.0 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6279iter count: 1.0 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:35:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641331 41.793644 41.562778 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.393066 67.441406 67.376953 +util 96.500000 100.000000 91.000000 +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 531 : avg iteration time 49.88 seconds +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 531 : data loading time 0.00 +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 531 : avg forward pass time 14.02 +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 531 : backward pass time 10.13 +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 531 : optimizer step time 1.57 +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 531-------------------- +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:36:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641349 41.793655 41.562794 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.270996 67.421875 67.220703 +util 96.000000 99.000000 91.000000 +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 532 : avg iteration time 48.60 seconds +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 532 : data loading time 0.00 +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 532 : avg forward pass time 13.37 +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 532 : backward pass time 10.13 +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 532 : optimizer step time 1.58 +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 532-------------------- +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:37:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641409 41.793839 41.562778 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.421875 67.396484 +util 93.750000 99.000000 88.000000 +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 533 : avg iteration time 46.02 seconds +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 533 : data loading time 0.00 +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 533 : avg forward pass time 12.07 +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 533 : backward pass time 10.15 +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 533 : optimizer step time 1.57 +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 533-------------------- +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:37:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641446 41.793957 41.562801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.416016 67.363281 +util 94.000000 99.000000 87.000000 +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 534 : avg iteration time 45.91 seconds +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 534 : data loading time 0.00 +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 534 : avg forward pass time 12.03 +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 534 : backward pass time 10.12 +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 534 : optimizer step time 1.59 +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 534-------------------- +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:38:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641462 41.793957 41.562824 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.687500 107.623047 +util 64.250000 70.000000 57.000000 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 535 : avg iteration time 158.21 seconds +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 535 : data loading time 0.00 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 535 : avg forward pass time 70.51 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 535 : backward pass time 8.41 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 535 : optimizer step time 0.39 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 535-------------------- +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5801 iter count: 1.0 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5801 iter count: 1.0 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5801 iter count: 1.0 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5801iter count: 1.0 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:41:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641455 41.793964 41.562801 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.319336 67.382812 67.296875 +util 94.500000 99.000000 89.000000 +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 536 : avg iteration time 47.10 seconds +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 536 : data loading time 0.00 +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 536 : avg forward pass time 12.62 +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 536 : backward pass time 10.14 +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 536 : optimizer step time 1.58 +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 536-------------------- +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:42:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641461 41.793957 41.562809 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.368164 67.460938 67.335938 +util 92.500000 98.000000 88.000000 +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 537 : avg iteration time 48.61 seconds +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 537 : data loading time 0.00 +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 537 : avg forward pass time 13.38 +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 537 : backward pass time 10.14 +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 537 : optimizer step time 1.57 +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 537-------------------- +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:42:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641470 41.793991 41.562809 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402344 67.539062 67.355469 +util 94.000000 99.000000 88.000000 +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 538 : avg iteration time 45.92 seconds +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 538 : data loading time 0.00 +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 538 : avg forward pass time 12.04 +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 538 : backward pass time 10.13 +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 538 : optimizer step time 1.57 +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 538-------------------- +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:43:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641596 41.794495 41.562809 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.456055 67.519531 67.433594 +util 95.750000 99.000000 92.000000 +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 539 : avg iteration time 48.52 seconds +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 539 : data loading time 0.00 +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 539 : avg forward pass time 13.33 +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 539 : backward pass time 10.15 +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 539 : optimizer step time 1.57 +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 539-------------------- +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:44:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641447 41.793781 41.562798 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.707031 107.525391 +util 66.250000 72.000000 60.000000 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 540 : avg iteration time 160.52 seconds +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 540 : data loading time 0.00 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 540 : avg forward pass time 71.65 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 540 : backward pass time 8.47 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 540 : optimizer step time 0.40 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 540-------------------- +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6133 iter count: 1.0 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6133 iter count: 1.0 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6133 iter count: 1.0 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6133iter count: 1.0 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:47:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641644 41.794529 41.562809 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.120117 68.373047 67.363281 +util 94.500000 99.000000 89.000000 +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 541 : avg iteration time 49.63 seconds +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 541 : data loading time 0.00 +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 541 : avg forward pass time 13.89 +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 541 : backward pass time 10.13 +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 541 : optimizer step time 1.58 +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 541-------------------- +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:48:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641687 41.794529 41.562939 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.086426 68.373047 67.226562 +util 96.250000 100.000000 91.000000 +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 542 : avg iteration time 45.96 seconds +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 542 : data loading time 0.00 +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 542 : avg forward pass time 12.06 +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 542 : backward pass time 10.14 +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 542 : optimizer step time 1.57 +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 542-------------------- +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:48:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641758 41.794529 41.562916 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.101074 68.392578 67.226562 +util 95.000000 99.000000 89.000000 +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 543 : avg iteration time 45.98 seconds +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 543 : data loading time 0.00 +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 543 : avg forward pass time 12.06 +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 543 : backward pass time 10.15 +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 543 : optimizer step time 1.57 +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 543-------------------- +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:49:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641851 41.794781 41.562939 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.086426 68.373047 67.226562 +util 94.500000 99.000000 88.000000 +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 544 : avg iteration time 48.54 seconds +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 544 : data loading time 0.00 +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 544 : avg forward pass time 13.35 +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 544 : backward pass time 10.15 +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 544 : optimizer step time 1.56 +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 544-------------------- +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:50:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641664 41.793880 41.562916 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.648438 107.544922 +util 66.250000 74.000000 60.000000 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 545 : avg iteration time 160.53 seconds +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 545 : data loading time 0.00 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 545 : avg forward pass time 71.65 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 545 : backward pass time 8.46 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 545 : optimizer step time 0.39 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 545-------------------- +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6289 iter count: 1.0 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6289 iter count: 1.0 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6289 iter count: 1.0 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6289iter count: 1.0 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:53:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641708 41.793903 41.562920 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.348633 67.500000 67.296875 +util 96.500000 99.000000 91.000000 +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 546 : avg iteration time 47.11 seconds +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 546 : data loading time 0.00 +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 546 : avg forward pass time 12.63 +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 546 : backward pass time 10.13 +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 546 : optimizer step time 1.57 +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 546-------------------- +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:53:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641802 41.794220 41.562920 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.333984 67.382812 67.316406 +util 94.250000 100.000000 87.000000 +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 547 : avg iteration time 43.35 seconds +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 547 : data loading time 0.00 +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 547 : avg forward pass time 10.77 +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 547 : backward pass time 10.11 +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 547 : optimizer step time 1.55 +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 547-------------------- +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641799 41.794209 41.562920 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373047 67.421875 67.355469 +util 96.000000 100.000000 91.000000 +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 548 : avg iteration time 48.49 seconds +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 548 : data loading time 0.00 +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 548 : avg forward pass time 13.32 +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 548 : backward pass time 10.14 +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 548 : optimizer step time 1.57 +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 548-------------------- +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:55:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641963 41.794731 41.563038 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358398 67.421875 67.335938 +util 94.250000 99.000000 89.000000 +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 549 : avg iteration time 48.62 seconds +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 549 : data loading time 0.00 +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 549 : avg forward pass time 13.39 +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 549 : backward pass time 10.13 +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 549 : optimizer step time 1.57 +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 549-------------------- +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:56:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641992 41.794823 41.563023 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.687500 107.603516 +util 67.250000 73.000000 60.000000 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 550 : avg iteration time 155.14 seconds +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 550 : data loading time 0.00 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 550 : avg forward pass time 68.98 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 550 : backward pass time 8.41 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 550 : optimizer step time 0.39 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 550-------------------- +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5459 iter count: 1.0 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5459 iter count: 1.0 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5459 iter count: 1.0 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5459iter count: 1.0 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:58:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642035 41.794895 41.563026 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.709961 67.806641 67.421875 +util 94.250000 99.000000 88.000000 +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 551 : avg iteration time 44.41 seconds +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 551 : data loading time 0.00 +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 551 : avg forward pass time 11.28 +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 551 : backward pass time 10.17 +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 551 : optimizer step time 1.57 +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 551-------------------- +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 05:59:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642062 41.794895 41.563057 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.056641 68.294922 67.343750 +util 93.250000 99.000000 88.000000 +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 552 : avg iteration time 46.08 seconds +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 552 : data loading time 0.00 +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 552 : avg forward pass time 12.08 +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 552 : backward pass time 10.18 +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 552 : optimizer step time 1.58 +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 552-------------------- +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:00:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642152 41.795166 41.563137 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.085938 68.314453 67.402344 +util 94.500000 99.000000 88.000000 +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 553 : avg iteration time 46.08 seconds +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 553 : data loading time 0.00 +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 553 : avg forward pass time 12.12 +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 553 : backward pass time 10.13 +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 553 : optimizer step time 1.58 +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 553-------------------- +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:01:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642176 41.795147 41.563232 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.085938 68.314453 67.402344 +util 96.250000 99.000000 90.000000 +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 554 : avg iteration time 46.03 seconds +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 554 : data loading time 0.00 +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 554 : avg forward pass time 12.08 +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 554 : backward pass time 10.15 +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 554 : optimizer step time 1.57 +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 554-------------------- +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:01:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641937 41.794083 41.563282 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.667969 107.583984 +util 64.500000 71.000000 57.000000 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 555 : avg iteration time 160.58 seconds +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 555 : data loading time 0.00 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 555 : avg forward pass time 71.69 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 555 : backward pass time 8.42 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 555 : optimizer step time 0.39 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 555-------------------- +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5684 iter count: 1.0 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5684 iter count: 1.0 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5684 iter count: 1.0 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5684iter count: 1.0 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:04:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641934 41.794102 41.563282 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.685547 67.767578 67.441406 +util 94.000000 99.000000 89.000000 +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 556 : avg iteration time 47.14 seconds +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 556 : data loading time 0.00 +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 556 : avg forward pass time 12.65 +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 556 : backward pass time 10.13 +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 556 : optimizer step time 1.57 +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 556-------------------- +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:05:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642068 41.794594 41.563282 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.724609 67.767578 67.597656 +util 94.500000 100.000000 88.000000 +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 557 : avg iteration time 48.70 seconds +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 557 : data loading time 0.00 +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 557 : avg forward pass time 13.41 +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 557 : backward pass time 10.16 +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 557 : optimizer step time 1.57 +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 557-------------------- +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:06:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642078 41.794594 41.563324 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.719727 67.767578 67.578125 +util 95.000000 99.000000 90.000000 +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 558 : avg iteration time 43.47 seconds +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 558 : data loading time 0.00 +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 558 : avg forward pass time 10.80 +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 558 : backward pass time 10.15 +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 558 : optimizer step time 1.57 +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 558-------------------- +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:06:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642158 41.794949 41.563309 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.719727 67.767578 67.578125 +util 94.750000 99.000000 89.000000 +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 559 : avg iteration time 45.94 seconds +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 559 : data loading time 0.00 +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 559 : avg forward pass time 12.05 +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 559 : backward pass time 10.13 +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 559 : optimizer step time 1.55 +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 559-------------------- +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:07:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641876 41.793709 41.563301 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.589844 107.583984 +util 63.250000 69.000000 56.000000 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 560 : avg iteration time 158.13 seconds +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 560 : data loading time 0.00 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 560 : avg forward pass time 70.49 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 560 : backward pass time 8.39 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 560 : optimizer step time 0.39 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 560-------------------- +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5664 iter count: 1.0 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5664 iter count: 1.0 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5664 iter count: 1.0 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5664iter count: 1.0 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:10:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641900 41.793732 41.563362 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.768066 67.845703 67.539062 +util 93.000000 99.000000 87.000000 +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 561 : avg iteration time 47.09 seconds +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 561 : data loading time 0.00 +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 561 : avg forward pass time 12.62 +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 561 : backward pass time 10.15 +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 561 : optimizer step time 1.57 +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 561-------------------- +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:11:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642075 41.794300 41.563377 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.090332 68.314453 67.421875 +util 95.750000 100.000000 91.000000 +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 562 : avg iteration time 48.68 seconds +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 562 : data loading time 0.00 +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 562 : avg forward pass time 13.38 +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 562 : backward pass time 10.18 +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 562 : optimizer step time 1.57 +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 562-------------------- +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:11:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642293 41.795135 41.563393 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.119629 68.353516 67.421875 +util 93.250000 99.000000 88.000000 +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 563 : avg iteration time 43.36 seconds +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 563 : data loading time 0.00 +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 563 : avg forward pass time 10.76 +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 563 : backward pass time 10.13 +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 563 : optimizer step time 1.57 +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 563-------------------- +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:12:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642290 41.795158 41.563385 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.197754 68.392578 67.617188 +util 94.500000 99.000000 88.000000 +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 564 : avg iteration time 46.07 seconds +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 564 : data loading time 0.00 +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 564 : avg forward pass time 12.11 +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 564 : backward pass time 10.12 +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 564 : optimizer step time 1.56 +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 564-------------------- +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:13:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641886 41.793530 41.563389 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.225098 108.423828 107.628906 +util 62.500000 68.000000 55.000000 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 565 : avg iteration time 157.98 seconds +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 565 : data loading time 0.00 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 565 : avg forward pass time 70.42 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 565 : backward pass time 8.39 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 565 : optimizer step time 0.39 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 565-------------------- +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5405 iter count: 1.0 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5405 iter count: 1.0 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5405 iter count: 1.0 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5405iter count: 1.0 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:15:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.641984 41.793869 41.563389 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.880371 68.001953 67.519531 +util 94.750000 100.000000 89.000000 +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 566 : avg iteration time 47.12 seconds +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 566 : data loading time 0.00 +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 566 : avg forward pass time 12.64 +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 566 : backward pass time 10.14 +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 566 : optimizer step time 1.56 +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 566-------------------- +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:16:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642042 41.794136 41.563389 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.875488 68.001953 67.500000 +util 94.500000 100.000000 87.000000 +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 567 : avg iteration time 43.49 seconds +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 567 : data loading time 0.00 +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 567 : avg forward pass time 10.81 +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 567 : backward pass time 10.14 +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 567 : optimizer step time 1.58 +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 567-------------------- +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:17:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642460 41.795647 41.563393 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.860840 68.001953 67.441406 +util 94.250000 99.000000 87.000000 +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 568 : avg iteration time 46.00 seconds +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 568 : data loading time 0.00 +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 568 : avg forward pass time 12.08 +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 568 : backward pass time 10.13 +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 568 : optimizer step time 1.57 +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 568-------------------- +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:18:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642471 41.795670 41.563393 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.880371 68.001953 67.519531 +util 94.500000 99.000000 89.000000 +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 569 : avg iteration time 46.12 seconds +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 569 : data loading time 0.00 +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 569 : avg forward pass time 12.10 +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 569 : backward pass time 10.15 +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 569 : optimizer step time 1.59 +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 569-------------------- +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:18:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642544 41.795933 41.563416 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.663574 107.726562 107.642578 +util 64.750000 71.000000 57.000000 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 570 : avg iteration time 160.69 seconds +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 570 : data loading time 0.00 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 570 : avg forward pass time 71.75 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 570 : backward pass time 8.40 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 570 : optimizer step time 0.39 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 570-------------------- +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6104 iter count: 1.0 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6104 iter count: 1.0 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6104 iter count: 1.0 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6104iter count: 1.0 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:21:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642466 41.795643 41.563416 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.676270 67.826172 67.226562 +util 93.000000 99.000000 87.000000 +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 571 : avg iteration time 50.27 seconds +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 571 : data loading time 0.00 +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 571 : avg forward pass time 14.22 +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 571 : backward pass time 10.13 +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 571 : optimizer step time 1.58 +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 571-------------------- +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:22:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642474 41.795643 41.563408 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.725098 67.884766 67.246094 +util 95.000000 99.000000 89.000000 +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 572 : avg iteration time 43.57 seconds +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 572 : data loading time 0.00 +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 572 : avg forward pass time 10.85 +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 572 : backward pass time 10.14 +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 572 : optimizer step time 1.57 +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 572-------------------- +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:23:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642472 41.795643 41.563438 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.983887 68.398438 67.845703 +util 93.500000 100.000000 88.000000 +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 573 : avg iteration time 46.03 seconds +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 573 : data loading time 0.00 +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 573 : avg forward pass time 12.08 +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 573 : backward pass time 10.14 +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 573 : optimizer step time 1.57 +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 573-------------------- +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:24:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642510 41.795666 41.563454 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.845703 67.246094 +util 93.000000 98.000000 89.000000 +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 574 : avg iteration time 43.45 seconds +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 574 : data loading time 0.00 +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 574 : avg forward pass time 10.79 +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 574 : backward pass time 10.16 +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 574 : optimizer step time 1.58 +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 574-------------------- +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:24:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642536 41.795765 41.563461 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.746094 107.525391 +util 65.000000 71.000000 56.000000 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 575 : avg iteration time 163.67 seconds +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 575 : data loading time 0.00 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 575 : avg forward pass time 73.22 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 575 : backward pass time 8.42 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 575 : optimizer step time 0.40 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 575-------------------- +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5732 iter count: 1.0 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5732 iter count: 1.0 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5732 iter count: 1.0 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5732iter count: 1.0 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:27:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642561 41.795860 41.563446 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.641602 67.806641 67.148438 +util 93.500000 99.000000 88.000000 +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 576 : avg iteration time 49.87 seconds +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 576 : data loading time 0.00 +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 576 : avg forward pass time 14.02 +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 576 : backward pass time 10.14 +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 576 : optimizer step time 1.56 +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 576-------------------- +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:28:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642559 41.795860 41.563446 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.149414 68.490234 67.128906 +util 94.750000 99.000000 89.000000 +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 577 : avg iteration time 43.37 seconds +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 577 : data loading time 0.00 +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 577 : avg forward pass time 10.75 +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 577 : backward pass time 10.13 +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 577 : optimizer step time 1.57 +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 577-------------------- +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:29:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642581 41.795856 41.563503 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.085938 68.314453 67.402344 +util 95.250000 100.000000 90.000000 +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 578 : avg iteration time 45.86 seconds +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 578 : data loading time 0.00 +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 578 : avg forward pass time 12.01 +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 578 : backward pass time 10.13 +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 578 : optimizer step time 1.57 +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 578-------------------- +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:29:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642672 41.796215 41.563507 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.071289 68.294922 67.402344 +util 95.000000 100.000000 88.000000 +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 579 : avg iteration time 46.09 seconds +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 579 : data loading time 0.00 +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 579 : avg forward pass time 12.13 +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 579 : backward pass time 10.12 +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 579 : optimizer step time 1.57 +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 579-------------------- +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:30:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642545 41.795723 41.563507 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.546387 107.609375 107.525391 +util 66.750000 74.000000 61.000000 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 580 : avg iteration time 160.93 seconds +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 580 : data loading time 0.00 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 580 : avg forward pass time 71.82 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 580 : backward pass time 8.43 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 580 : optimizer step time 0.40 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 580-------------------- +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5928 iter count: 1.0 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5928 iter count: 1.0 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5928 iter count: 1.0 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5928iter count: 1.0 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:33:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642555 41.795723 41.563534 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.368164 67.402344 67.355469 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 581 : avg iteration time 47.08 seconds +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 581 : data loading time 0.00 +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 581 : avg forward pass time 12.61 +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 581 : backward pass time 10.14 +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 581 : optimizer step time 1.56 +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 581-------------------- +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:34:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642577 41.795845 41.563522 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729492 67.845703 67.382812 +util 96.000000 99.000000 90.000000 +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 582 : avg iteration time 43.41 seconds +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 582 : data loading time 0.00 +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 582 : avg forward pass time 10.77 +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 582 : backward pass time 10.16 +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 582 : optimizer step time 1.56 +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 582-------------------- +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:34:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642588 41.795868 41.563511 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.690430 67.845703 67.226562 +util 95.250000 99.000000 90.000000 +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 583 : avg iteration time 46.09 seconds +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 583 : data loading time 0.00 +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 583 : avg forward pass time 12.12 +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 583 : backward pass time 10.14 +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 583 : optimizer step time 1.57 +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 583-------------------- +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:35:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642704 41.796352 41.563511 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.739258 67.845703 67.421875 +util 96.250000 100.000000 90.000000 +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 584 : avg iteration time 45.90 seconds +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 584 : data loading time 0.00 +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 584 : avg forward pass time 12.03 +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 584 : backward pass time 10.13 +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 584 : optimizer step time 1.57 +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 584-------------------- +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:36:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642649 41.796120 41.563530 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.609375 107.603516 +util 64.000000 70.000000 56.000000 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 585 : avg iteration time 163.18 seconds +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 585 : data loading time 0.00 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 585 : avg forward pass time 73.02 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 585 : backward pass time 8.39 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 585 : optimizer step time 0.39 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 585-------------------- +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6152 iter count: 1.0 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6152 iter count: 1.0 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6152 iter count: 1.0 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6152iter count: 1.0 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:38:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642654 41.796143 41.563530 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.347656 67.382812 67.335938 +util 94.250000 100.000000 87.000000 +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 586 : avg iteration time 44.39 seconds +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 586 : data loading time 0.00 +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 586 : avg forward pass time 11.26 +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 586 : backward pass time 10.16 +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 586 : optimizer step time 1.58 +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 586-------------------- +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:39:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642693 41.796154 41.563625 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.441406 67.335938 +util 95.000000 99.000000 90.000000 +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 587 : avg iteration time 43.31 seconds +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 587 : data loading time 0.00 +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 587 : avg forward pass time 10.73 +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 587 : backward pass time 10.13 +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 587 : optimizer step time 1.58 +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 587-------------------- +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:40:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642685 41.796154 41.563629 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.376953 67.441406 67.355469 +util 95.500000 99.000000 91.000000 +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 588 : avg iteration time 46.06 seconds +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 588 : data loading time 0.00 +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 588 : avg forward pass time 12.10 +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 588 : backward pass time 10.13 +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 588 : optimizer step time 1.57 +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 588-------------------- +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:41:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642751 41.796394 41.563644 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.479492 67.492188 67.441406 +util 95.750000 99.000000 91.000000 +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 589 : avg iteration time 46.00 seconds +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 589 : data loading time 0.00 +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 589 : avg forward pass time 12.08 +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 589 : backward pass time 10.13 +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 589 : optimizer step time 1.58 +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 589-------------------- +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:41:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642602 41.795765 41.563667 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.648926 107.726562 107.623047 +util 66.750000 73.000000 59.000000 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 590 : avg iteration time 157.87 seconds +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 590 : data loading time 0.00 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 590 : avg forward pass time 70.35 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 590 : backward pass time 8.40 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 590 : optimizer step time 0.39 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 590-------------------- +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6035 iter count: 1.0 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6035 iter count: 1.0 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6035 iter count: 1.0 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6035iter count: 1.0 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:44:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642642 41.795815 41.563763 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.324707 67.343750 67.318359 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 591 : avg iteration time 47.18 seconds +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 591 : data loading time 0.00 +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 591 : avg forward pass time 12.67 +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 591 : backward pass time 10.13 +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 591 : optimizer step time 1.58 +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 591-------------------- +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:45:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642656 41.795818 41.563770 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.734863 67.865234 67.343750 +util 95.500000 99.000000 89.000000 +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 592 : avg iteration time 48.60 seconds +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 592 : data loading time 0.00 +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 592 : avg forward pass time 13.39 +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 592 : backward pass time 10.14 +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 592 : optimizer step time 1.55 +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 592-------------------- +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:46:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642678 41.795815 41.563770 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.710449 67.865234 67.246094 +util 94.500000 100.000000 89.000000 +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 593 : avg iteration time 46.08 seconds +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 593 : data loading time 0.00 +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 593 : avg forward pass time 12.13 +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 593 : backward pass time 10.12 +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 593 : optimizer step time 1.57 +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 593-------------------- +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:46:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642678 41.795815 41.563770 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.710449 67.865234 67.246094 +util 95.250000 100.000000 89.000000 +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 594 : avg iteration time 43.36 seconds +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 594 : data loading time 0.00 +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 594 : avg forward pass time 10.75 +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 594 : backward pass time 10.13 +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 594 : optimizer step time 1.57 +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 594-------------------- +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:47:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642420 41.794746 41.563782 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.628906 107.583984 +util 63.750000 70.000000 56.000000 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 595 : avg iteration time 160.75 seconds +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 595 : data loading time 0.00 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 595 : avg forward pass time 71.79 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 595 : backward pass time 8.41 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 595 : optimizer step time 0.39 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 595-------------------- +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5898 iter count: 1.0 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5898 iter count: 1.0 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5898 iter count: 1.0 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5898iter count: 1.0 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:50:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642423 41.794746 41.563789 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.144043 68.314453 67.636719 +util 95.500000 100.000000 89.000000 +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 596 : avg iteration time 44.41 seconds +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 596 : data loading time 0.00 +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 596 : avg forward pass time 11.30 +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 596 : backward pass time 10.13 +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 596 : optimizer step time 1.56 +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 596-------------------- +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:51:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642613 41.795517 41.563805 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.100098 68.314453 67.460938 +util 96.500000 99.000000 91.000000 +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 597 : avg iteration time 43.43 seconds +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 597 : data loading time 0.00 +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 597 : avg forward pass time 10.79 +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 597 : backward pass time 10.12 +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 597 : optimizer step time 1.57 +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 597-------------------- +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:51:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642620 41.795506 41.563789 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.217285 68.333984 67.871094 +util 96.250000 99.000000 92.000000 +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 598 : avg iteration time 48.49 seconds +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 598 : data loading time 0.00 +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 598 : avg forward pass time 13.33 +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 598 : backward pass time 10.13 +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 598 : optimizer step time 1.56 +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 598-------------------- +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:52:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.642644 41.795506 41.563881 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.119629 68.314453 67.539062 +util 95.500000 100.000000 90.000000 +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 599 : avg iteration time 48.73 seconds +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 599 : data loading time 0.00 +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 599 : avg forward pass time 13.44 +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 599 : backward pass time 10.14 +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 599 : optimizer step time 1.57 +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 599-------------------- +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 7, 2026 - 06:53:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 7, 2026 - 06:53:27 | INFO | fastgen.trainer:run:225 ] Training complete. +[May 7, 2026 - 06:53:27 | INFO | fastgen.trainer:validate:358 ] Validation iteration 600 +[May 7, 2026 - 06:53:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:53:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:53:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.458 +[May 7, 2026 - 06:53:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:53:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:53:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.548 +[May 7, 2026 - 06:53:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:53:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:53:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.873 +[May 7, 2026 - 06:53:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:53:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:53:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.301 +[May 7, 2026 - 06:53:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:53:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:53:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=3.781 +[May 7, 2026 - 06:53:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:53:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:53:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.081 +[May 7, 2026 - 06:53:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:54:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:54:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.546 +[May 7, 2026 - 06:54:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:54:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:54:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.183 +[May 7, 2026 - 06:54:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:54:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:54:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.987 +[May 7, 2026 - 06:54:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 7, 2026 - 06:54:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 7, 2026 - 06:54:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.100 +[May 7, 2026 - 06:54:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 600-------------------- +[May 7, 2026 - 06:54:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 7, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 600 +[May 7, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.286 (n=10) +[May 7, 2026 - 06:54:35 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 600 +[May 7, 2026 - 06:54:35 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints/0000600 +[May 7, 2026 - 06:54:55 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 600 +[May 7, 2026 - 06:54:55 | INFO | fastgen.trainer:run:232 ] Taking a 10 sec nap and exiting training. +[May 7, 2026 - 06:55:05 | INFO | fastgen.utils.distributed:clean_up:88 ] Distributed clean up. +[May 7, 2026 - 06:55:05 | INFO | __main__::49 ] Training finished. diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/requirements.txt b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/wandb-metadata.json b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d22927f28eb86704bfc2767ec234cc74a29f5d57 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/wandb-metadata.json @@ -0,0 +1,70 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-06T10:10:16.306844Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_2step_t937_no_reward.py", + "-", + "trainer.resume=False", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_2step_t937_no_reward", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1165820178432" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "mtg4sjqss4j2qo4zq4jrh08h43c2gge7" +} \ No newline at end of file diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/wandb-summary.json b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..13edca6bc2bd7657d2d5b20cb4f07a07f71dfb81 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/files/wandb-summary.json @@ -0,0 +1 @@ +{"val0/sync_c_sample_8":5.986749172210693,"fake_score/trainable_params":1421383840,"val0/sync_c_sample_6":6.546391487121582,"fake_score/total_params":1421383840,"train/vsd_loss":0.58984375,"train/gan_loss_disc":0,"val0/generated":{"_type":"videos","count":10,"videos":[{"caption":"sync_c=6.458","_type":"video-file","sha256":"fc14a16bac32611ba42f99ca44e4dcffd0c53658bfdb7ddfc3700f2612c5418c","size":209477,"path":"media/videos/val0/generated_600_fc14a16bac32611ba42f.mp4"},{"caption":"sync_c=5.548","_type":"video-file","sha256":"86f8df0cf9c178a9bc205c736298720c84f30c56ccf5c1bc42e7a3c8fb8a22b9","size":208037,"path":"media/videos/val0/generated_600_86f8df0cf9c178a9bc20.mp4"},{"size":200192,"path":"media/videos/val0/generated_600_8ae4464b03faf92d8fcc.mp4","caption":"sync_c=4.873","_type":"video-file","sha256":"8ae4464b03faf92d8fcc6b1be690fc26fd2fc1be9cd0c51f1a6d6b41c12f0176"},{"size":291439,"path":"media/videos/val0/generated_600_90413783c59d55141104.mp4","caption":"sync_c=4.301","_type":"video-file","sha256":"90413783c59d55141104700f624ed9a70a38de17983d65c487cb0a3c2862d7c4"},{"sha256":"2dc2540d510c8648dc000abc1321c2d78ecb3fb3d2a9c46bb012554c2fdb2447","size":247598,"path":"media/videos/val0/generated_600_2dc2540d510c8648dc00.mp4","caption":"sync_c=3.781","_type":"video-file"},{"caption":"sync_c=7.081","_type":"video-file","sha256":"c2ea37b78886ecef23af2a167f3009211e1399bcddc93094b457c746162fcb10","size":182572,"path":"media/videos/val0/generated_600_c2ea37b78886ecef23af.mp4"},{"sha256":"9db48cbc78876e078e80e5ebdd7dfbd361e2bba79bb1b9c9e0a93fc4c58ffa51","size":288888,"path":"media/videos/val0/generated_600_9db48cbc78876e078e80.mp4","caption":"sync_c=6.546","_type":"video-file"},{"caption":"sync_c=5.183","_type":"video-file","sha256":"ea52c20a941a74fc52e88ca271d58442d20dbfaf05582f041a367bc867edb3db","size":193724,"path":"media/videos/val0/generated_600_ea52c20a941a74fc52e8.mp4"},{"caption":"sync_c=5.987","_type":"video-file","sha256":"3af3ff96013a3f3a3e810246a9f2ac50eafa426ffb9e9842c5584db69acdd52a","size":235006,"path":"media/videos/val0/generated_600_3af3ff96013a3f3a3e81.mp4"},{"caption":"sync_c=3.100","_type":"video-file","sha256":"02c1aae0e691f96ea5ad5306c71f17c454dfc1623713fe1753165d72a108a174","size":236276,"path":"media/videos/val0/generated_600_02c1aae0e691f96ea5ad.mp4"}],"captions":["sync_c=6.458","sync_c=5.548","sync_c=4.873","sync_c=4.301","sync_c=3.781","sync_c=7.081","sync_c=6.546","sync_c=5.183","sync_c=5.987","sync_c=3.100"]},"model/trainable_params":2842767680,"val0/total_loss":0,"val0/sync_c_sample_3":4.301151275634766,"train/gan_loss_gen":0,"_step":600,"val0/reconstructed":{"videos":[{"size":182028,"path":"media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4","_type":"video-file","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327"},{"sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214","size":170922,"path":"media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4","_type":"video-file"},{"_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8","size":173236,"path":"media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4"},{"path":"media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4","_type":"video-file","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372},{"sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004,"path":"media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4","_type":"video-file"},{"sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630,"path":"media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4","_type":"video-file"},{"_type":"video-file","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986,"path":"media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4"},{"_type":"video-file","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516,"path":"media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4"},{"sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001,"path":"media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4","_type":"video-file"},{"_type":"video-file","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490,"path":"media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4"}],"captions":false,"_type":"videos","count":10},"train_media/student/generation":{"size":198813,"path":"media/videos/train_media/student/generation_500_6a6b80f57451f4c65363.mp4","_type":"video-file","sha256":"6a6b80f57451f4c65363131082571e6a60ea0a82948c057269716527cf91b416"},"profiler/avg_forward_pass_time":13.438956025056541,"profiler/backward_pass_time":10.137195223942399,"optimizer/iteration":600,"profiler/data_loading_time":0.000958496704697609,"model/local_total_params":[4285560952,4283762808,4283762808,4283762808],"_runtime":42287.49742507,"_wandb":{"runtime":42287},"train/vsd_loss_unweighted":0.58984375,"val0/sync_c_sample_5":7.081058502197266,"val0/sync_c_sample_9":3.100022315979004,"val0/sync_c_sample_1":5.548144340515137,"_timestamp":1.7781044755176215e+09,"optimizer/non_finite_grads_count (model_key net)":0,"net/trainable_params":1421383840,"optimizer/lr_fake_score":2e-06,"model/local_trainable_params":[711111248,710552144,710552144,710552144],"train_media/data/real":{"size":190461,"path":"media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4","_type":"video-file","sha256":"2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae"},"optimizer/grad_norm (model_key net)":0,"profiler/optimizer_step_time":1.5748728550970554,"profiler/avg_iteration_time":48.732473373413086,"train/fake_score_loss":0.00832366943359375,"val0/sync_c_sample_0":6.457763671875,"train/total_loss":0.00832366943359375,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"optimizer/lr_net":2e-06,"model/total_params":17136849376,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"train/vsd_loss_weighted":0.58984375,"val0/sync_c_sample_4":3.780697822570801,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"net/total_params":1421383840,"net/local_total_params":[355555624,355276072,355276072,355276072],"val0/sync_c_sample_2":4.872723579406738,"val0/sync_c_sample_7":5.1834397315979,"val0/sync_c_mean":5.285814189910889} \ No newline at end of file diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug-core.log b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..c5414d6794aed6d9245c772bca01ec223d85737d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-06T19:10:16.687704156+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdndgnwy6/port-3167398.txt","pid":3167398,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-06T19:10:16.68862858+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3167398} +{"time":"2026-05-06T19:10:16.688170889+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3167398-3203462-121902078/socket","Net":"unix"}} +{"time":"2026-05-06T19:10:16.780123545+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-06T19:10:16.798605358+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"c4sg4krp","id":"1(@)"} +{"time":"2026-05-06T19:10:17.444331933+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"c4sg4krp","id":"1(@)"} +{"time":"2026-05-06T19:10:23.686203823+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k1fk7jo8hsk9"} +{"time":"2026-05-07T06:55:05.903619023+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-07T06:55:05.903827468+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-07T06:55:05.903821439+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-07T06:55:05.903871947+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-07T06:55:05.903913164+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3167398-3203462-121902078/socket","Net":"unix"}} +{"time":"2026-05-07T06:55:07.722225874+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-07T06:55:07.722245955+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-07T06:55:07.72225495+09:00","level":"INFO","msg":"server is closed"} diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug-internal.log b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..cf8097dd2c52fc2162d709c1a3b71a8647371d3d --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-05-06T19:10:16.799200194+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-06T19:10:17.443315611+09:00","level":"INFO","msg":"stream: created new stream","id":"c4sg4krp"} +{"time":"2026-05-06T19:10:17.444248699+09:00","level":"INFO","msg":"handler: started","stream_id":"c4sg4krp"} +{"time":"2026-05-06T19:10:17.444325118+09:00","level":"INFO","msg":"stream: started","id":"c4sg4krp"} +{"time":"2026-05-06T19:10:17.444355369+09:00","level":"INFO","msg":"writer: started","stream_id":"c4sg4krp"} +{"time":"2026-05-06T19:10:17.444354811+09:00","level":"INFO","msg":"sender: started","stream_id":"c4sg4krp"} +{"time":"2026-05-06T22:52:10.007450117+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/paulhcho/OmniAvatar-FastGen/c4sg4krp/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-07T06:55:05.903830281+09:00","level":"INFO","msg":"stream: closing","id":"c4sg4krp"} +{"time":"2026-05-07T06:55:07.347145243+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-07T06:55:07.720260545+09:00","level":"INFO","msg":"handler: closed","stream_id":"c4sg4krp"} +{"time":"2026-05-07T06:55:07.720330233+09:00","level":"INFO","msg":"sender: closed","stream_id":"c4sg4krp"} +{"time":"2026-05-07T06:55:07.720346042+09:00","level":"INFO","msg":"stream: closed","id":"c4sg4krp"} diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug.log b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3fe01544b66f4bcae1e9d87c1e4ca4c5c53c3ca1 --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_setup.py:_flush():81] Configure stats pid to 3167398 +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug.log +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/logs/debug-internal.log +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:init():844] calling init triggers +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_2step_t937_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '600', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-06 19:10:16,308 INFO MainThread:3167398 [wandb_init.py:init():892] starting backend +2026-05-06 19:10:16,780 INFO MainThread:3167398 [wandb_init.py:init():895] sending inform_init request +2026-05-06 19:10:16,794 INFO MainThread:3167398 [wandb_init.py:init():903] backend started and connected +2026-05-06 19:10:16,799 INFO MainThread:3167398 [wandb_init.py:init():973] updated telemetry +2026-05-06 19:10:16,819 INFO MainThread:3167398 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-06 19:10:18,404 INFO MainThread:3167398 [wandb_init.py:init():1042] starting run threads in backend +2026-05-06 19:10:18,680 INFO MainThread:3167398 [wandb_run.py:_console_start():2524] atexit reg +2026-05-06 19:10:18,680 INFO MainThread:3167398 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-06 19:10:18,680 INFO MainThread:3167398 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-06 19:10:18,680 INFO MainThread:3167398 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-06 19:10:18,686 INFO MainThread:3167398 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-07 06:55:05,903 INFO wandb-AsyncioManager-main:3167398 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-07 06:55:05,903 INFO wandb-AsyncioManager-main:3167398 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/run-c4sg4krp.wandb b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/run-c4sg4krp.wandb new file mode 100644 index 0000000000000000000000000000000000000000..01685149a40bc2d116a47119157a4d906ca0ac3a --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260506_191016-c4sg4krp/run-c4sg4krp.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eccce9f5179260bcc5f553bf51672747d8f28d7fead2bacecc3fa9a3943538f4 +size 9382015 diff --git a/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb_id.txt b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..792b38f99dbda4ec571c5b32416017a3c2b9d0fe --- /dev/null +++ b/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb_id.txt @@ -0,0 +1 @@ +c4sg4krp