diff --git a/.gitattributes b/.gitattributes index aa29c97935bfaf044965bd93b2e9a36d01f747dc..4fa82a617899565c546651e6a6f08cd4d24123fa 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12286,3 +12286,446 @@ hyunbin_FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/d hyunbin_FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/wandb/run-20260426_020257-xgrmxmeb/files/media/videos/val_gt/videos_0_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text hyunbin_FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/wandb/run-20260426_020257-xgrmxmeb/files/media/videos/val_gt/videos_0_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text hyunbin_FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/wandb/run-20260426_020257-xgrmxmeb/run-xgrmxmeb.wandb filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_100_850d3119537af39cd73a.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_200_6aeb862b86389a224bd0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_300_5f8cb46048abdbaac380.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_400_e795ee614555844015bf.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_500_cfcf07eca45d7088f701.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_0b75f5edcf051be4d644.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_47880f13ffec08641064.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_4c1d4f3b53c89063c461.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_4e3326a320f59b4f9e51.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_7b4f819f49f849872726.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_8fa30017029df47665dd.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_b3a4fc44b16ea3e27813.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_cbfb982d2111fa764411.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_dd84c54f83ae42112e9b.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_f93cf9999b324b2e7a7b.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_225e957587ae1d0fe7d5.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_2d3b42afc78bf79ca449.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_4390895edd06743e8c72.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_546c80c6dc91bb30f5cb.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_56195b749da49d777495.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_919f21445f552e1c6f12.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_94c8e13c20dc57814ecc.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_972ff665996bdad61f95.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_d5bdd032f7934dda45fc.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_d645f569a0ef081bd030.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_02c5b3593e2cffd5435d.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_12dd3e8b68539c186c45.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_5ac9b349fb9b2f7aff87.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_6d02c6ba0aefb48d5f21.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_849f252a7cf7498be320.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_87e264f834b51c38f1b7.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_9812fc648ae6293a18f2.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_9d859d9798bc03af9ebc.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_c13b21d72c692c27829b.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_ee66c421b880cd4f91c5.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_18af891940de05d9b331.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_2e499a7dda694b010cec.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_3e90c5033c55af20bb6e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_411567b23b2a9d11f8b0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_73310431574902993fc3.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_7bf153bb435b3c38f5b1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_809ee63db512071dfe0a.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_84c67faa0c475909ba32.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_ae9928d9057006762cdf.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_bcaa6a3de515fd5911e0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_27992951a72b38f5d81f.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_46b5cb00d4ffc57ce4ee.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_504a52ea94c1a40c46ae.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_5a8e19f0eb99243bf2a0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_74697543046d4e78858e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_81e7c8a2f8d15eb1e275.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_966c91cef37ce41ca528.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_96f6e6139b2cb00023bb.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_b97f2844fc731a551443.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_c8d17d8470213e2fede7.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_00cb74bae01fa7368a46.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_134b51494130bcd535cf.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_1b22e3065fc30b1fc3cb.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_3c513fbe04eea74399aa.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_3e07d13ddaace09f352a.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_96c9100dc7b878101ec0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_c7944e8bf03d0b8b4341.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_c8e42f6e6ca899b7d68d.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_d3b3c38cc4cea459a9ba.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_e305c2ec1a356386ea3a.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/run-b1ilcpvk.wandb filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/run-9ojgh41i.wandb filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_BRANCH_700_TO_800_FRESHOPT/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_700to800_freshopt/checkpoints/0000700.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_604_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_700_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_800_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_604_b620abcbaca8ba7db5ae.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_700_7f42501ced78a5981448.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_800_6a56be7cdd484957da32.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_499e72e8cf7a95b7ce2a.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_6cfbfdbe06a9c9de0d70.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_7153810c5617a19a2452.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_7bfbf18b4144b09e0ca1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_c684aef24e40913d3d79.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_d7768a50b6800410273d.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_e1326f3bb435a7877876.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_e3e5e12e7f91d944d305.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_ec97738b316e0805b191.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_ff4e60c3023112769d09.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_0f9daabfb34bdab144f8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_167b48c370d4888a11a8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_1c516bb19a99114dc64d.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_2c3f00aec8ff488467cd.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_9786c381ce111ca85f04.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_9a8077b0f50d01138495.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_a6c4aa554d4fa1df2714.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_d22b6a96303085aabe4a.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_de624079a2f729993c17.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_f8d024dde8f9dfcbc78e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_1df1b740de2839d7049d.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_3cb47282a0c6e17cd468.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_662b4e119e06fcfed243.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_9b8ee95c989012245e95.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_a5a88c03e4f38bb03812.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_adbe9f7f90fb3f2f5d21.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_c60f017da4163f5d875e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_c721fe88bdc87fbf3873.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_e186c4b95df8e54e6a23.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_f4aa7cb18093a4c80225.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_0899e8e33a3daa243303.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_44d664992b102e09aee6.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_564155b491ace85b197b.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_5ff7b05030a5d107b25c.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_6050d77106d97831a984.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_7c40be6e5027d8c71d06.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ac8f091c51ecec2b900d.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_b6a1e5b3f1258ffa6f86.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ba7693264ef940cbfab8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ec6cb67cea81ecc6beb8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/run-lvwg9pfc.wandb filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_042b02b2ceeb648cf655.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_1bd4125209338a80be7e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_1d5d847d7164bb28616f.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_4b40b35b2d64012475c8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7105cc2283e6acf8678a.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_776855bea09e3ee28d4f.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7a8d3fc9242239c85404.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7f1688b26c115ba620a5.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_cdda7c097236ecac61ec.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_e55b0b718380e13aab7d.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/run-lvwg9pfc.wandb filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_18cdb0fb0acea943ae1b.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_56f78017d0746eb5d2dc.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_59adee4c2c1233294521.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_6853c5e28978367eb479.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_75bb737f23f37c356994.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_8180e794591d55c93416.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_85b5b43e4c471767517f.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_b2a3a97608a2bdd39582.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_b4cb8b6f832fd8a17929.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_dfe41c776e29223e9b8c.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/run-lw2bm1l6.wandb filter=lfs diff=lfs merge=lfs -text +FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/run-fxrufsq8.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9493e950c25fba8a49c168bcd82a3668c5be3c7c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd21c1c424930fcb1020a8d3bc3fdce5f25fe741d60ebe33c913ebd8fc9c6192 +size 635542 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1644baec3d7df334acb55a33d62c508e88d17d81 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebaf4130b00ec0457afe145caeadcca432b52a0ad95f23308e870038f8ec6921 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a816f2fb269e0c00c491442a0a1c330e5f5602ec --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53d4e5b2d1adaecb9ead40ac6695da682c2642aa2fe37ce741bb0d2308def7c9 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fb2a34ffcdb94c9d36566fed64f476a2a7d9874d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:069a3242cb11b890a6255f4e9a4930b4069e176a87c6fddf0dd066ea416c9b75 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7ca2d735a7df407ff0eb61d9b67d93068bf22989 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:708b39829054e0af563244ad119f2e99ec4f4a1d5bc0c9c7bc43eb9fec8edacd +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..51c2d08a0198125df9c822e82420ab041b0c12cb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:791cc166a7f7b61e600c1d233928252493e247cc1a0bbdc4f68e5fd32a1c124a +size 635535 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..16de716d8dbbf3b16681deb0701896d19a9d978e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4d46bb1e57d863993193d7e94af351830b9cfac91c4a0e2c696320f1af0841b +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d7ad9d249b6019b8b0be626718ae54edf1705536 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9421d251b41e8920f9ef57fee4157cc51d18cad33da0a1a63d4754aea2ea787 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..01de9eb3bc9b63ffe75f380e26bab4ea6861d680 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ba08b5bd87fba04b2aa498a7ed8e47ea8f2f821ab0d0fbafd93871dff8fe88 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..270cec998efd33dd64c1e00bbd38e8f4f2ea01a7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c558f30ce44c0161fb75eaf8bed787320d8087257b73617b9521bb96caf059f +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f637e0f6792d8ae1960e0e448573321eacbddc4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9e7bf7b483afea9b5f5fc9f6d1a8368eea2246e5e5404c08b9fba8acdc0064 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6b37d0a5c9ef736fa53aafad0911e9fb3bb121fa --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53c856dddadabd24f8b07c241eb511df9d8b5dd13b90b13238adeed69d3bca42 +size 635542 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..265766d0d4cffc3a9c421a65533b10c06b8bad22 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac6899cd7e118c64a9d18bfb1b7d572dd3c0f270c4dec6056d92c50f5782fc91 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f1b0ed5e284efd69911b28d46184662857c2dcb7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6a7ad4615aac3ff8e2b4855df4bb245c499216b11e52c1e8a5de3e715937776 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c7be10f4c108f5dd1279c9570e4e3e602ee41fea --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a25913156943dbf3c4171c850dc2318c527ae3098e16d513019ce799ba7607a5 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ae39e7ee34ced98c270f6620688e41ea410fbe8c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc3ecdcc10d50b12c1be183052385f8d7e493403cb137369f83744d124902408 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..111527d57cac5c010cff72ffab9a4aa8c671da9b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57471961225a6aeb7c2f6984bbf31b46c2b830e290b5ebdac2030f05a81fc088 +size 635535 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..10003b4fe9ed4b0c08dbae34f73316a6e07b0e17 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38a63883a97c2a0159e29a5a1d9630ed696dc0e2b27d57f864cebee1799bcbfd +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..172d2e8d7dbef7a270564c5ada68771912416573 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:304e203391a9ffc5c5fd345f3d92c12a9575820f9def14691ac539b3af492f19 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..743a688b10f42293eba46169cadec6a0cf1aef57 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f9e4455c7ebac0085f2bdf0f3b71bd355fd8ee75144fef755739f610a06bfe1 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..11c45a318fea339d6e4330ae2602ae528e92bf4a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffb89403cece783a21205ad0faa71a201007fa66cb744379d3f34eeaa1fab7d5 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.pth new file mode 100644 index 0000000000000000000000000000000000000000..94689c6d88777305ec14be3e82dcf676c28c81a1 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d49c29dddf4cff9abe84c331dfad17799824bf4158c145f77812c5430228f1 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6231682ac87f914040308bc3e433e6c15459b02c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89d260dff0a0fea9610bdef08b18d91b6d8320189e166c7e80c1e79f3ba0133 +size 635542 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..529a4ef6f569dd2f7b76cedf27c0251889a6193a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1600b1236b8712379ac40cc4f52ebc3a23b7fea96716bf0a46c153a14b0aea7 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2220eab860b529dcfc8fa3039a9771f8e045e5ff --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b143ce70dbc87eb6eb590db24971d9ae220de8d22870687587adb6c324331d82 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7b819523649e47bd33a25e22df31be52c2289e98 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff004b8c6c892b41df6360c025e59311b8adba45704ae037a9c86be1c5c8892a +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..83a16cd5ef60726420e397b3cbed0a077ba84ff3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:025f8448437e872fa2bfda8217adbdbf60bf4e8e3dc320efd1a95e1620f04cda +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1c487cc4a26dec10b7b78bfe99f4c27e2f24ff2a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40c30fe224ff91978b062df8dd43e7f6f59a0aae747f41f923b05a8dae87cb03 +size 635535 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..333839712c577f665ac4251a762fe8558bcfe731 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10d75c64b35d0f259a1503b600a7bc94df689c80c4010bd1fb30e8aff12d3131 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e95805e0494ca11cdd2412cbeb6b652c9654b7c8 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff230974467c54989e90ff6283af1df814fafc72d7c8a3fa141b9fbe78bb07f9 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7c757d1b47c0a89c6a76632ef6882a46279c97dd --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4190bcc4108481269c54caab1097af3091ea3a53c2269a7fb4cb22bbf224beb +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e97cf8f79e5492572313c5b0eda4fd90c56cd796 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c68e498b4f76b0167925cb415b8cf163e3b77298b00138fba5b1f9ca5fcc5393 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.pth new file mode 100644 index 0000000000000000000000000000000000000000..36cdfef63708c11e993e5164fd9859eb64d6d33e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463104a57d6bfde142e5cd01a059afdb8b99dad40c73e04a236502ee34fb4d66 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ae7bdede2f61985e9f1eef83e52fb0aa89d6df4f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ff37cf3fc313fbb8d90b3f09a5bfed5a25c52ee224f9e262d2a646d79d7b17c +size 635542 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d3a8324300de23963807637213fc0d333d75a775 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a7db2523d7b258f22bbe09bfb40170fe3bb7010e0d7cfb8c70cb0a5de14d462 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bd8a83aa4a3d48ad9c6bf27c6a1f8f86db735404 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4ab62f0a61659d942d64e89b42d1eb0f6e4942d97242b4c40072825ce558065 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..305cd085a83878c3eb9ff62b02889ae88d6e417e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:455fab784212fbb87c653187f1979cdd92e0647e7cf08c0f46722eee8bb4a75e +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d49027918a205bdb616261dcf422c057cf4cc34e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cf79479ca3c2d4a35aa45bdf3253565a3a8723a165086ad11e57a8d1b58308f +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..b1664c7320c0b5279479f652fd84d26ce7ba1341 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49682a4b868bf542df77fe50f9431959ab022dcbfa3c5c5bf3d582eaedf1e1f1 +size 635535 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e074d216ba021332ab5e209f2c6a9a81757e0f7b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8630627fb63bc09b3e96252f76fafe5304d4864edaee65088c84cee6e9ce3ade +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c59cd6e6e0073cd0c7cbd8b675f60136dcfb228f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b096d72f2fb720193149220c7911d6b45b4b66d8cef8efd42ade888fcd55daf1 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f99aa7beb788aba964ac1df2e27e67db87dc4af7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1838aed6132070efc0033e1edc38af81daa544d2440eec14def18e20fba96382 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d97863b073230baa12855e32c511006c59e55735 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d85740b3383d8950f23cbbed0b10c213cdd1e5debfc9d163813f2e2b5a3d5f77 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.pth new file mode 100644 index 0000000000000000000000000000000000000000..0deb1499fd51197d9815f644a93381fbe0678161 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31545bbbd2979dddf082789dbf999d9324dc9febf1b016304ddc713ede429c3e +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1a45ae365cba386b9dde09347fd3b06c07439d68 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7efceac034867acf8e64deb3e3ccab40743afa7aaca4b27df9b79df3d6b1f98 +size 635542 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6ae2ec5039c552dd852505ebd79210e3ba1c3827 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:380c4bc6276f81015b1a2afaee8196e5236889465f38e84a9fff944249aaa32c +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..eeaa204f3adee30b129659ba5ea8dfff5dcf7c30 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96a37b9f3e4b525996a5adb3d21a5fa2d3bcfab650932e6120dbdc6216ed9433 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..57bee1b89771af968b7f7139b337fa8c72fb8dd8 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b9a5df6029b646dd0b9c7cb17073462c73feea2a41ae8fa6b7cbdd434fc6349 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ec091a2bb59ae09273f4f8998ea1ed19d51e2314 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e139f4a9dafcd7b13459d74daa32ba81214a71112c101a15b2d97929eb829851 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ade3eeb909f2d2142bbdc5706925fc99d68bf6c7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85df807980f357637263388b9fcfcf9bfda0ebe456dbd3b4521fcc17a732062f +size 635535 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c88f12ac392513832f4fed9ffc8f1f72cafed287 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10810b18f6f7147735ffa89088edb4d383d11cd2480951615de869ff092589e2 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..efee7b1ecffd5ba21f75f643277a3ce037265e3d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b9eb4dbb412e099c38f2bbe125bca3fa7adac03d2d66ba0b73373db73eb5c84 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..906d7c5dcb00cc18f2818e2d9c703de078257318 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:608d5e2350f0d120574711e99d9fa978061b056b0e4405af0eec348da416cbaa +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..70a5226ec6c74500c30933331ed4ccaa88249954 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5720f0b8ff0bc751010d7543ffcaf65934b7e2528ee524c23fa95c76c798e84 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..df0fb483027e40ac33d6a36f4b38fcfd244d6a44 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2110994f1f90ec15f3f831251e70d29fe1b83f8a35bb969069c4ac3f211c6d +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..3f400d455a97de397cbb07808d3681d6dc5f84f1 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b895840989f8950b9faa1ec4e98241b6f025da5c9917eb22ddf74156916efce3 +size 635542 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8c703d740a02fffebb81db0e13aa5f9cbafe034b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:885899e44822bef32566966a87ccd6d627ebf40f727a19925c97bd5dafd7e89b +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bbe7963d7072112d5d1092db17ff2603889b4c40 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e495e6d51cfa57d38b869eeddb58776f2ef957c2af622f7e9ce31bd5aa6e3e4c +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0b8f0dac1e25ca2cd8ae69bea1fd20b15abdc606 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4f8e18a61cccf82c36350a78e0b3ac65959c47f977fca8694dea9cbac133142 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a461e9e22c6776cbffbf2ed5e8383a9f9c4efa4c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e42ea5c3891246bd967f8a3953a728426f54d62d976e00eb61ba85d408d719c +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ce5692923c9c732423d1641d74719839300cf27b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e0831a6c226ce1e6aa6c35af83b3b79c992b52f86b872aec5c67984ee11bfbe +size 1722906 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..5979108b10c2f3775c7217a3eb2f8bc002f8fb2d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98c89772262079ca2d19a98dacf055e5dfbee095439a32e7828aa4d91b33a750 +size 2847149355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8b4ca7451f7ce4ef0137ff584c3ba631bfe3b1a2 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9b332258db0d8d65950a1536f8f13d24a80ebae7e9c6895a87b64cc47c4c4d8 +size 2845267132 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6740fb41aa09715b10a3f07a863ec07d2de565c9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69a8db79d468c57d108f81283dac90cc47eb8f58e15dedb93b249b7fbaf994d2 +size 2845352958 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3a8bef63902059c9ee4d5b3a9275376b4f991384 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:796e00fd8dfc59f01bcd13ae84b4c62759e84ef58ab2c67fbed5c76a235fafe8 +size 2845316241 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..baa667736c70140446437a53373acb7ac3042c90 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3278adbe8c47f0187f87f77c3aab43ff02e50e024b273193f97372ff97770a5f +size 635535 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..cb567d393ba9fe27086881b2f21b17b509ba60ae --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5514d7add1acb38c21a0bd642bbd6c6a624ff3cf3d67bf2183e45b81865a2a4 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4981a84247cbad1f903bf36f9dfb7582a3ce4498 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e511cd2ee31e361c23f6c8275cad97a2f32772badce8734cdfda05f58944231b +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1bc9321b9ceffbf8f456e6d169b89cc7682cdc28 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9c69e5b9eb012cc535aa71b6524528889965de1e850e2f43158493571ada41e +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..69ed3bab7e9b4533eeed08c9aca383f12e7149f1 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80adf74991d8398a419f9ad121b5b20605ba81f2cd071288aa05583c2b262842 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..a58b8459209cda0677679254bee0a26a0405ea9b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c71899782ad38e07229365308473894714d9da695ac6e3243a487a7c998bd8 +size 1722749 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..dce8f588522154c4e037e2bf2a6de8ea6477b808 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35bbbb7aba8ab15facbcb8a368e12620dad3aa53d0f6c8694fff7554bf4ad834 +size 2847149355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..04b02124b47f6e1ddb2a44c3044746eba21fe5f9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b0c1528452a6f594f9bebb50162ff6633d7db1e04bfed52512befb19d6b0666 +size 2845267132 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..df6eb0d723ced8196f4c8c6846435946296128b1 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c13ab0fb967b89460350991299d1796aea704bec504ab2b1aa7aa9f6bae698a +size 2845352958 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ccf5a5f958322bae48eee01f6586306f103ff21c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c896a9bf5c2c97d9fb987a82b92f5f8e987da0f1c3efbe859a2a50f2d9855b6f +size 2845316241 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.pth new file mode 100644 index 0000000000000000000000000000000000000000..97cb41dfed21f440317a176863edebc119d1a324 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c1c25fd6c3ea2819196abcf7cba095dde0d86545b36426cd731e4feee3caa8 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ac47afa47242f569b3b31fc006d51d6c3c5b399 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_2step_t624_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'False' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '0' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.624' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '600' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..90a70ee67d959267ccdc9cc04b348ad81d229a59 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-04T13:51:22.529921257+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-04T13:51:23.09264564+09:00","level":"INFO","msg":"stream: created new stream","id":"b1ilcpvk"} +{"time":"2026-05-04T13:51:23.093286905+09:00","level":"INFO","msg":"handler: started","stream_id":"b1ilcpvk"} +{"time":"2026-05-04T13:51:23.093343757+09:00","level":"INFO","msg":"stream: started","id":"b1ilcpvk"} +{"time":"2026-05-04T13:51:23.093361998+09:00","level":"INFO","msg":"sender: started","stream_id":"b1ilcpvk"} +{"time":"2026-05-04T13:51:23.093384534+09:00","level":"INFO","msg":"writer: started","stream_id":"b1ilcpvk"} +{"time":"2026-05-05T01:33:03.996515354+09:00","level":"INFO","msg":"stream: closing","id":"b1ilcpvk"} +{"time":"2026-05-05T01:33:05.171530649+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T01:33:05.471966099+09:00","level":"INFO","msg":"handler: closed","stream_id":"b1ilcpvk"} +{"time":"2026-05-05T01:33:05.472037644+09:00","level":"INFO","msg":"sender: closed","stream_id":"b1ilcpvk"} +{"time":"2026-05-05T01:33:05.472054626+09:00","level":"INFO","msg":"stream: closed","id":"b1ilcpvk"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f1b97c94f78f7f5a9877153f51be6bb0b52e2281 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/debug.log @@ -0,0 +1,21 @@ +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_setup.py:_flush():81] Configure stats pid to 493121 +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug.log +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug-internal.log +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_init.py:init():844] calling init triggers +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_2step_t624_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '600', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-04 13:51:22,025 INFO MainThread:493121 [wandb_init.py:init():892] starting backend +2026-05-04 13:51:22,510 INFO MainThread:493121 [wandb_init.py:init():895] sending inform_init request +2026-05-04 13:51:22,524 INFO MainThread:493121 [wandb_init.py:init():903] backend started and connected +2026-05-04 13:51:22,528 INFO MainThread:493121 [wandb_init.py:init():973] updated telemetry +2026-05-04 13:51:22,557 INFO MainThread:493121 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-04 13:51:23,816 INFO MainThread:493121 [wandb_init.py:init():1042] starting run threads in backend +2026-05-04 13:51:24,244 INFO MainThread:493121 [wandb_run.py:_console_start():2524] atexit reg +2026-05-04 13:51:24,244 INFO MainThread:493121 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-04 13:51:24,244 INFO MainThread:493121 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-04 13:51:24,244 INFO MainThread:493121 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-04 13:51:24,262 INFO MainThread:493121 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 01:33:03,996 INFO wandb-AsyncioManager-main:493121 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 01:33:03,996 INFO wandb-AsyncioManager-main:493121 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31845ddd432ad2a3b00583166db4b969eaf07233 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/config.yaml @@ -0,0 +1,388 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + btht0sri7vddjdptc2alnflsboni5uhl: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_2step_t624_no_reward.py + - '-' + - trainer.resume=False + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_2step_t624_no_reward + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1175943299072" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward + startedAt: "2026-05-04T04:51:22.023532Z" + writerId: btht0sri7vddjdptc2alnflsboni5uhl + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_2step_t624_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "False" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "0" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.624" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "600" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9684c7bde9ed18086d39e7a9880b642c505b3947 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112c0905448a821974f0bf0a94d9107473bb02b60e0ebe1e75a57d309670facb +size 219311 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e38036c10e966c2c82fa717b29431a1ff88e1ae --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae +size 190461 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_100_850d3119537af39cd73a.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_100_850d3119537af39cd73a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ea1a7391b2954785666140f6c659b2be7a9ad3d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_100_850d3119537af39cd73a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:850d3119537af39cd73a9517e4ad3a33ef9e2c5d238befb567cd542d54cfbc0c +size 188086 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_200_6aeb862b86389a224bd0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_200_6aeb862b86389a224bd0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..deda896301f68f14e88bebd8a46b46a7019843e9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_200_6aeb862b86389a224bd0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeb862b86389a224bd0225efdffde9c7d3ec386696bbfce94fa38e17568baf2 +size 189990 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_300_5f8cb46048abdbaac380.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_300_5f8cb46048abdbaac380.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a72a9019fe0349e2b9910e30f2ed0f5e02f1874 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_300_5f8cb46048abdbaac380.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f8cb46048abdbaac38003e4f41896b542ddce55faaee17ea886f2ac5b424dca +size 212162 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_400_e795ee614555844015bf.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_400_e795ee614555844015bf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9559796afd9161b2668feca587b9fc353447f231 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_400_e795ee614555844015bf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e795ee614555844015bf07691a94a8fb38b3fdae34e49ba7d60069f12006074b +size 232230 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_500_cfcf07eca45d7088f701.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_500_cfcf07eca45d7088f701.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4b82115765bafb90808fefb7cdced3b185bc939 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/train_media/student/generation_500_cfcf07eca45d7088f701.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfcf07eca45d7088f701b448ea711950d0a54a6f3ef852b62eea2308234c436c +size 213362 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_0b75f5edcf051be4d644.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_0b75f5edcf051be4d644.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aeecc785d0672f1930152211f64587436c304095 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_0b75f5edcf051be4d644.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b75f5edcf051be4d6444028c53274babd66de2bb2020fc12d9045d1e6cf7915 +size 188017 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_47880f13ffec08641064.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_47880f13ffec08641064.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..baeffda22230e95ccf896638568b54b39bca2795 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_47880f13ffec08641064.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47880f13ffec08641064b4016300c8b9a8072ff096510c1a91fb089642ae7648 +size 260027 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_4c1d4f3b53c89063c461.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_4c1d4f3b53c89063c461.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e996c947ef61ba25477ade1e4ea3c659c420fbfa --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_4c1d4f3b53c89063c461.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c1d4f3b53c89063c4611f93e6a01b203c0235bf92202eb0d6c3505c02f66588 +size 180706 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_4e3326a320f59b4f9e51.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_4e3326a320f59b4f9e51.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..12ed9f61b857c621f9b9f601f0013faccadb5bb8 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_4e3326a320f59b4f9e51.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e3326a320f59b4f9e512884c25e8ada5c8880bffef8b886fdd104e06a54228a +size 213647 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_7b4f819f49f849872726.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_7b4f819f49f849872726.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3581630ce6d7dc01e9ebec1b0f8ef7e2cc5a064f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_7b4f819f49f849872726.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b4f819f49f849872726b8273f8c2e10ae106ecf0222a865e5a68242344f1e13 +size 175304 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_8fa30017029df47665dd.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_8fa30017029df47665dd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c424912fee1978fa23011ce512f28fa405b0dcf4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_8fa30017029df47665dd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fa30017029df47665dd31a1777161eac70ebab5d367a4d70e43f0fa403380d9 +size 212344 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_b3a4fc44b16ea3e27813.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_b3a4fc44b16ea3e27813.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ffd5b6f5cd7457e826cf7cfd7bf2137b9f591f3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_b3a4fc44b16ea3e27813.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a4fc44b16ea3e278130075601f3383399b10ac304067949716ade834993524 +size 213199 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_cbfb982d2111fa764411.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_cbfb982d2111fa764411.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63b5b082fb7f65327e59060069a77a32a3163697 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_cbfb982d2111fa764411.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbfb982d2111fa7644113790d4ffa04ab860982666d41cb37b495164475b45c2 +size 160242 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_dd84c54f83ae42112e9b.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_dd84c54f83ae42112e9b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c098bf3d4aae33bb740df8632ab4fc9c5d13ef1e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_dd84c54f83ae42112e9b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd84c54f83ae42112e9bfc77f3f933f564c6cf27f5ab84546c07b0824e02096d +size 170175 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_f93cf9999b324b2e7a7b.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_f93cf9999b324b2e7a7b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7066eb71c150bd15a4e3a2fd56cd13f4f862d72 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_100_f93cf9999b324b2e7a7b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f93cf9999b324b2e7a7ba6b035d1cb0180511696eee63943b9039408fd624d26 +size 241151 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_225e957587ae1d0fe7d5.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_225e957587ae1d0fe7d5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e759fb28a18c7ee404fa58433ebc96e5a0c9d2b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_225e957587ae1d0fe7d5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:225e957587ae1d0fe7d5afb8966eca0dee7048377810587a183da172e90369e4 +size 165198 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_2d3b42afc78bf79ca449.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_2d3b42afc78bf79ca449.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ec2096c79450ff9350e68dab5603410b41a4f8a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_2d3b42afc78bf79ca449.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d3b42afc78bf79ca449b9a2ecc79a254bf0560d7d5896944b7192b916a2665b +size 250580 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_4390895edd06743e8c72.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_4390895edd06743e8c72.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0f3058899f468542cae7955796347f39f9a72d2a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_4390895edd06743e8c72.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4390895edd06743e8c72f129877cb1dbfae51cbf03a90e971553c76ac27cd493 +size 188356 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_546c80c6dc91bb30f5cb.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_546c80c6dc91bb30f5cb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ce830b4918008030c48d051d4f0b087d27183b58 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_546c80c6dc91bb30f5cb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:546c80c6dc91bb30f5cbae2caf459e5949f9d0d4632a925d901eb8d9c43a7dee +size 182287 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_56195b749da49d777495.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_56195b749da49d777495.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64cae1599d2186beffbc930b2079b76bd89b4b4b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_56195b749da49d777495.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56195b749da49d777495afc2c163929d9f3b59a9578b601d678985ede1c68a2b +size 220588 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_919f21445f552e1c6f12.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_919f21445f552e1c6f12.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a2a4e8ad0c980b525617f86fe2ad701d69537294 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_919f21445f552e1c6f12.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:919f21445f552e1c6f1255c3361dadf9b7e04dbb4b4a949c60e09bbab03cdbc2 +size 218352 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_94c8e13c20dc57814ecc.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_94c8e13c20dc57814ecc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fdbbe035dd6fd90f661b4a846e7565438d3081cb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_94c8e13c20dc57814ecc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94c8e13c20dc57814eccd8ac4c228a6d310fba2939a704067bae421cb17d07b7 +size 265239 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_972ff665996bdad61f95.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_972ff665996bdad61f95.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4bb301fd1cb3d88f4e5e68c18cb9798daa51ec6c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_972ff665996bdad61f95.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:972ff665996bdad61f95bf9b7c7ad87effa69324839ce213eacc35f8ea79155b +size 193784 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_d5bdd032f7934dda45fc.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_d5bdd032f7934dda45fc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cad9756f0f17388ff1404d1d01f86341e1c8ec67 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_d5bdd032f7934dda45fc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5bdd032f7934dda45fca593a08ec5d638c6e5019ede9f5a4018096990c8970e +size 221991 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_d645f569a0ef081bd030.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_d645f569a0ef081bd030.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7121cdceb266018a8cefd55ccffdbe6daa9b03ca --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_200_d645f569a0ef081bd030.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d645f569a0ef081bd0307438b388de006ff38bf62367e00d20a145526c21148d +size 177557 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_02c5b3593e2cffd5435d.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_02c5b3593e2cffd5435d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..020f8c2faaca354da472ee6ff8700305778860f6 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_02c5b3593e2cffd5435d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02c5b3593e2cffd5435db0c21f8ddc5dac3e7be96a7963e40acac567672fbd2e +size 194403 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_12dd3e8b68539c186c45.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_12dd3e8b68539c186c45.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d81366b404af80330275b0d3efa73ccb5ac3634f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_12dd3e8b68539c186c45.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12dd3e8b68539c186c45c01012d6ab9bb073f95f831442155059dc5274ed8f34 +size 204454 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_5ac9b349fb9b2f7aff87.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_5ac9b349fb9b2f7aff87.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ba845a86d1164acf90399f0b9ff10fafcd5fc4d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_5ac9b349fb9b2f7aff87.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ac9b349fb9b2f7aff87bffffc591f7b656a3776740da05f2fe97163830bc0a7 +size 228816 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_6d02c6ba0aefb48d5f21.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_6d02c6ba0aefb48d5f21.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..56187400ae0e2f7c921c8546cae94719b1efb879 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_6d02c6ba0aefb48d5f21.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d02c6ba0aefb48d5f2180c447a547bf2baee15b6dfaf8a3b7564fb7b2e1f6e0 +size 204608 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_849f252a7cf7498be320.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_849f252a7cf7498be320.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9eb3a0b9d7b333063e3b9ca37497a2554d612444 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_849f252a7cf7498be320.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:849f252a7cf7498be320df56daf2a596dcb93caa3f97f5a0600fea3dd8c41ff0 +size 237250 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_87e264f834b51c38f1b7.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_87e264f834b51c38f1b7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9801fc3638d2803878af369201c528490ebccb3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_87e264f834b51c38f1b7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87e264f834b51c38f1b731d4c60b283de1014834e0d5a2b26b717147d2448ea0 +size 176185 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_9812fc648ae6293a18f2.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_9812fc648ae6293a18f2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f637d33ef3a805f13da6e1ec9ac759d0cdc96fbb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_9812fc648ae6293a18f2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9812fc648ae6293a18f27d83ce877a21fd8e8eccde788ca1688c869a52a67e65 +size 233646 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_9d859d9798bc03af9ebc.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_9d859d9798bc03af9ebc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3f461e3986de8ee4add02660e239c233b61d449f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_9d859d9798bc03af9ebc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d859d9798bc03af9ebc604537f2fc7718ee8927e9865b9f42b58206207ef9b5 +size 188977 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_c13b21d72c692c27829b.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_c13b21d72c692c27829b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d803aeb159b6475f77f08c2c93a5f3cd559e07c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_c13b21d72c692c27829b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c13b21d72c692c27829bc33c15added1e363802cb2d29c048a993ae9191b2ba9 +size 269094 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_ee66c421b880cd4f91c5.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_ee66c421b880cd4f91c5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9b581a022496cabbb2f379b9c25b3d2563590c51 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_300_ee66c421b880cd4f91c5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee66c421b880cd4f91c5f6a3d62bcf68adb74f382a8cf29a65649ea6b2533ac4 +size 285610 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_18af891940de05d9b331.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_18af891940de05d9b331.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ae89b4a7eca1876e4f24eea6f2618f22e878ee00 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_18af891940de05d9b331.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18af891940de05d9b3316106976a9bbfb45d2a2b351d33d555d452b2738c5cd6 +size 195846 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_2e499a7dda694b010cec.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_2e499a7dda694b010cec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1c467c7a01ec2ddbae73481dcf81aa977d1eddcb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_2e499a7dda694b010cec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e499a7dda694b010cec34a308ab05557b1730a2d0155ba3ccd6a868e5263c43 +size 227204 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_3e90c5033c55af20bb6e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_3e90c5033c55af20bb6e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6597933076ac33fd6fb4836dc3ef190eee20b8dd --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_3e90c5033c55af20bb6e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e90c5033c55af20bb6ece24a6e7d564fd0ac8f6fe873abc27af58f227f48fa8 +size 267975 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_411567b23b2a9d11f8b0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_411567b23b2a9d11f8b0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68bf46166e9f0c90e3cae9a8ae6aefab28b0a427 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_411567b23b2a9d11f8b0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:411567b23b2a9d11f8b001ec9ff52a0efc3eda6412ef786aa3181260d6a2a2d2 +size 228649 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_73310431574902993fc3.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_73310431574902993fc3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e38d2147f1cbd404a7fc214fc7e722c0022adbc5 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_73310431574902993fc3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73310431574902993fc371c2eb8757b079858dfac2231c3b595263c9c0eda75f +size 202116 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_7bf153bb435b3c38f5b1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_7bf153bb435b3c38f5b1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8cf25c90bd6a53abc04b4bf38e18d33fa6fce96b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_7bf153bb435b3c38f5b1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bf153bb435b3c38f5b18b19ccf909b9a7b261b63c58761151d070172dac3d3f +size 234844 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_809ee63db512071dfe0a.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_809ee63db512071dfe0a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5939dec1d0117ae8e235f4d61b7efc32a8d89599 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_809ee63db512071dfe0a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:809ee63db512071dfe0afaee1a726dde11656849f74ba095bef18b20396b73d9 +size 277429 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_84c67faa0c475909ba32.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_84c67faa0c475909ba32.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6bc228ffe51afa398b4eea378f36b866ceb88c13 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_84c67faa0c475909ba32.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c67faa0c475909ba32d120121d3a8c4be9a41aaa4a76562e1d8db8a45a5b8b +size 176410 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_ae9928d9057006762cdf.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_ae9928d9057006762cdf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ba705afec1bb3e8db5baf1ac0aee8d7805d1403 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_ae9928d9057006762cdf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae9928d9057006762cdf2af6ee97c984065dae695b8ab0a368456d5d0c4a3988 +size 200699 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_bcaa6a3de515fd5911e0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_bcaa6a3de515fd5911e0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..679a25933f2a3aeb16690d603a52d0427b753730 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_400_bcaa6a3de515fd5911e0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcaa6a3de515fd5911e012e7bb3992a7f5508a980b8b89fde8d2a54f626627d8 +size 186145 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_27992951a72b38f5d81f.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_27992951a72b38f5d81f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5b3c6b2475106dfeaa6ee9f5a4dad1beee697cc0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_27992951a72b38f5d81f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27992951a72b38f5d81f0f9154581f33cf5080744e191c6e9e135c7114afb105 +size 207464 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_46b5cb00d4ffc57ce4ee.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_46b5cb00d4ffc57ce4ee.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33db006092ca798df159a048774fd40c40c99f77 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_46b5cb00d4ffc57ce4ee.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46b5cb00d4ffc57ce4ee57063ef4fca2a7550ecb87e4e8584de41d823216ca6b +size 239432 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_504a52ea94c1a40c46ae.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_504a52ea94c1a40c46ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cab06dcd42a6fa1bf734b77c8c8164764a8698f9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_504a52ea94c1a40c46ae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:504a52ea94c1a40c46ae31925fcc1dd4d4241b68368abc48c78a1d0eef288a60 +size 182219 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_5a8e19f0eb99243bf2a0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_5a8e19f0eb99243bf2a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cd732631084d8739842b68d1ec19d4673e22221c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_5a8e19f0eb99243bf2a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a8e19f0eb99243bf2a0418334e2bd42c411cf205ef020ec0305d6250853109d +size 201453 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_74697543046d4e78858e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_74697543046d4e78858e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..098ff32d425c92085bd722bd9cf3241c2e63571e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_74697543046d4e78858e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74697543046d4e78858ed0b29a9451b5a82ff24f6039dc3a59db88286d85c402 +size 234178 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_81e7c8a2f8d15eb1e275.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_81e7c8a2f8d15eb1e275.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..681eeaa843335cf7f54ffa60c6ecdf49803f01f1 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_81e7c8a2f8d15eb1e275.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81e7c8a2f8d15eb1e275d16d97f7a2c27b3436874e033f7ef384ba0b2f9947b0 +size 247440 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_966c91cef37ce41ca528.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_966c91cef37ce41ca528.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6124ff557dafc69fb7ac69e8b4d805a1e3981071 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_966c91cef37ce41ca528.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:966c91cef37ce41ca5282428926ce1732da730321bbb7d519f8df19a1b17c80b +size 289303 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_96f6e6139b2cb00023bb.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_96f6e6139b2cb00023bb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..623a7e496e7c993e0f971cdb6f2c50e06d6951fe --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_96f6e6139b2cb00023bb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96f6e6139b2cb00023bb710837e535ebcb15942c830d8ee3505cbe559a1e365f +size 193488 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_b97f2844fc731a551443.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_b97f2844fc731a551443.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48d771787c1da9ac69ae29a4a04a53de66d5b64f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_b97f2844fc731a551443.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b97f2844fc731a5514434e27a12d2f69c188a13f738ff84d1da9abcddfeaa231 +size 282915 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_c8d17d8470213e2fede7.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_c8d17d8470213e2fede7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f473316734970ccf7f06c7ee15c09135bab0049 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_500_c8d17d8470213e2fede7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8d17d8470213e2fede7e3df7e76b2d0f0f85086396f7f3e7d4ee305aea4ec3c +size 210795 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_00cb74bae01fa7368a46.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_00cb74bae01fa7368a46.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab6b1e824fb17ee268e189727bdd2091a6013d68 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_00cb74bae01fa7368a46.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00cb74bae01fa7368a462766cf019fa01052af74b46f580f9b45b74664db14e0 +size 294125 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_134b51494130bcd535cf.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_134b51494130bcd535cf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f6db16eecf459cdad55de80a6f9f330cd88f851 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_134b51494130bcd535cf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:134b51494130bcd535cf515bed310a633a8b72c42f368eeb1862567657d0e0ef +size 184559 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_1b22e3065fc30b1fc3cb.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_1b22e3065fc30b1fc3cb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08c69cef20a1f2f3ad0caa342c564cdba949e106 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_1b22e3065fc30b1fc3cb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b22e3065fc30b1fc3cbeb5167f60ec66c005e488d63f4dec82349b8cbc26012 +size 214086 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_3c513fbe04eea74399aa.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_3c513fbe04eea74399aa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a972fb125b796a23d13d08719f5367b147b8b553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_3c513fbe04eea74399aa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c513fbe04eea74399aa394a92545729fab7216e103ecdae2fd35a6dcc58b645 +size 294516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_3e07d13ddaace09f352a.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_3e07d13ddaace09f352a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..037c1fd14947db82c76363c857e74456f0662885 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_3e07d13ddaace09f352a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e07d13ddaace09f352a05693b03db33cbb4b216740402c34f83c9f6206c14aa +size 245698 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_96c9100dc7b878101ec0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_96c9100dc7b878101ec0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d9f0df6bc8a2cda7f87798948979bc7a057fa70 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_96c9100dc7b878101ec0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96c9100dc7b878101ec0ab5f2a75fda367ee6f900907e3fb2cc2d2e03252e4a8 +size 240915 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_c7944e8bf03d0b8b4341.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_c7944e8bf03d0b8b4341.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f83ff149f4c970e55056b63b858c38602303726 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_c7944e8bf03d0b8b4341.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7944e8bf03d0b8b4341e0d7bbee5f4e7b0adc3d3330765fb30f75d0d5357645 +size 255214 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_c8e42f6e6ca899b7d68d.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_c8e42f6e6ca899b7d68d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..634c6c47fb148fb33edba49f05a353bc7e78eb3d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_c8e42f6e6ca899b7d68d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8e42f6e6ca899b7d68d4632aaad57bff35f1e5e8dee2c8bdc68d1a34ff2c5b7 +size 198675 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_d3b3c38cc4cea459a9ba.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_d3b3c38cc4cea459a9ba.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92e506466550dff9aa6ca228feb96e8cc200883e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_d3b3c38cc4cea459a9ba.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3b3c38cc4cea459a9ba1c021895364ca9ae872202d870a9df2e11f5fe734db0 +size 213374 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_e305c2ec1a356386ea3a.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_e305c2ec1a356386ea3a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07d5dffea73f7b3f01c8909b412a5a74ec60b167 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/generated_600_e305c2ec1a356386ea3a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e305c2ec1a356386ea3a46601e061d7c700229b33d09a7e02e7b462110835147 +size 208137 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/output.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c2df9a3a761850b094d8c381d1ad5f7de81053e7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/output.log @@ -0,0 +1,10736 @@ +[May 4, 2026 - 13:51:24 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 4, 2026 - 13:51:24 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 4, 2026 - 13:51:24 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 4, 2026 - 13:51:24 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 4, 2026 - 13:51:24 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 4, 2026 - 13:51:24 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 4, 2026 - 13:51:24 | INFO | fastgen.trainer:run:77 ] Starting training +[May 4, 2026 - 13:51:24 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 4, 2026 - 13:51:24 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 4, 2026 - 13:51:51 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 4, 2026 - 13:52:03 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 4, 2026 - 13:52:03 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 4, 2026 - 13:52:03 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 4, 2026 - 13:52:03 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 4, 2026 - 13:52:47 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 4, 2026 - 13:52:47 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 4, 2026 - 13:52:47 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 4, 2026 - 13:52:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 4, 2026 - 13:52:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 4, 2026 - 13:52:47 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 4, 2026 - 13:52:49 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 4, 2026 - 13:52:49 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 4, 2026 - 13:52:49 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 4, 2026 - 13:52:49 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 4, 2026 - 13:52:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 4, 2026 - 13:52:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.9s +[May 4, 2026 - 13:52:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 4, 2026 - 13:52:51 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 4, 2026 - 13:53:03 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 4, 2026 - 13:53:03 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.1s +[May 4, 2026 - 13:53:03 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 4, 2026 - 13:53:03 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 4, 2026 - 13:53:03 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 4, 2026 - 13:53:03 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[May 4, 2026 - 13:53:03 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17136.85 M total params (logical). +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4285.56 M total params LOCAL on rank 0. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 1. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 2. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 3. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 4, 2026 - 13:53:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 4, 2026 - 13:53:09 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.371752 37.433289 37.340736 +peak_gpu_mem_gb 57.011160 57.011160 57.011160 +peak_gpu_mem_reserved_gb 59.156250 59.156250 59.156250 +util 84.000000 85.000000 82.000000 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.37 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 10.33 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 10.13 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.74 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 13:53:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.547559 37.608700 37.516609 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 90.250000 92.000000 88.000000 +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 46.07 seconds +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.08 +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 10.18 +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 1.56 +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 13:54:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.547917 37.609100 37.517216 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 91.500000 99.000000 88.000000 +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 49.17 seconds +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 13.63 +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 10.19 +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 1.57 +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 13:55:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.548038 37.607899 37.518131 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.211426 61.212891 61.210938 +util 89.750000 91.000000 88.000000 +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 43.42 seconds +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 10.76 +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 10.17 +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 1.57 +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 13:56:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.569791 37.628223 37.540943 +peak_gpu_mem_gb 96.121632 96.123194 96.121111 +peak_gpu_mem_reserved_gb 102.568848 102.658203 102.539062 +util 57.250000 59.000000 55.000000 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 161.08 seconds +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 71.61 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 8.45 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.44 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8447 iter count: 1.0 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8447 iter count: 1.0 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8447 iter count: 1.0 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8447iter count: 1.0 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 13:58:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.570056 37.628502 37.540955 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.758789 64.820312 64.738281 +util 90.750000 99.000000 87.000000 +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 49.70 seconds +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 13.92 +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 10.14 +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 1.57 +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 13:59:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.570482 37.628811 37.541054 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656250 64.703125 64.640625 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 48.54 seconds +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 13.34 +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 10.16 +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 1.57 +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:00:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.570754 37.628967 37.541374 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.631836 64.664062 64.621094 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 48.63 seconds +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 13.38 +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 10.16 +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 1.56 +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 8-------------------- +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:01:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.570887 37.629375 37.541470 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.714844 64.761719 64.699219 +util 89.750000 91.000000 88.000000 +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 43.46 seconds +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 10.79 +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 10.12 +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 1.57 +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 9-------------------- +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:02:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.571751 37.630020 37.542194 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.728027 105.882812 105.263672 +util 61.250000 63.000000 59.000000 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 163.43 seconds +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 73.09 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 8.45 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.39 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 10-------------------- +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8223 iter count: 1.0 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8223 iter count: 1.0 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8223 iter count: 1.0 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8223iter count: 1.0 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:04:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.571894 37.630054 37.542545 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.680664 64.859375 64.621094 +util 99.000000 100.000000 98.000000 +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 47.09 seconds +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 12.60 +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 10.15 +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 1.57 +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 11-------------------- +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:05:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.572078 37.630173 37.542793 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.651367 64.742188 64.621094 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 48.62 seconds +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 13.39 +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 10.13 +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 1.57 +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 12-------------------- +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:06:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.572227 37.630642 37.542782 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.661133 64.781250 64.621094 +util 93.500000 99.000000 88.000000 +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 48.64 seconds +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 13.39 +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 10.14 +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 1.57 +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 13-------------------- +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:07:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.572850 37.630898 37.543106 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.622070 64.742188 64.582031 +util 92.000000 93.000000 90.000000 +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 46.06 seconds +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 12.11 +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 10.13 +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 1.57 +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 14-------------------- +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:08:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.572996 37.630074 37.543751 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.186035 105.263672 105.160156 +util 61.250000 63.000000 59.000000 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 158.20 seconds +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 70.49 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 8.40 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.39 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 15-------------------- +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7598 iter count: 1.0 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7598 iter count: 1.0 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7598 iter count: 1.0 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7598iter count: 1.0 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:10:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.573549 37.630123 37.545696 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 64.976562 64.875000 +util 95.750000 97.000000 94.000000 +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 44.44 seconds +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 11.30 +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 10.13 +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 1.57 +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 16-------------------- +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0062 iter count: 1.0 +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:11:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.573759 37.630611 37.545891 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871094 64.917969 64.855469 +util 96.000000 100.000000 86.000000 +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 46.08 seconds +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 12.10 +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 10.15 +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 1.57 +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 17-------------------- +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:12:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.574004 37.630711 37.545906 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749023 64.898438 64.699219 +util 94.500000 95.000000 93.000000 +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 46.11 seconds +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 12.12 +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 10.13 +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 1.57 +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 18-------------------- +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0062 iter count: 1.0 +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:12:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.574570 37.632545 37.546082 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.851562 64.898438 64.835938 +util 93.000000 94.000000 91.000000 +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 45.97 seconds +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 12.02 +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 10.19 +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 1.57 +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 19-------------------- +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0062 iter count: 1.0 +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:13:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.574728 37.632229 37.546444 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.195801 105.302734 105.160156 +util 59.250000 60.000000 57.000000 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 163.78 seconds +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 73.31 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 8.39 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.40 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 20-------------------- +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8965 iter count: 1.0 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8965 iter count: 1.0 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8965 iter count: 1.0 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8965iter count: 1.0 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:16:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.574781 37.632416 37.546432 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.808105 64.900391 64.777344 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 44.52 seconds +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 11.33 +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 10.14 +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 1.58 +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 21-------------------- +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:17:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.574943 37.632889 37.546524 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.676270 64.757812 64.431641 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 46.02 seconds +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 12.07 +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 10.16 +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 1.56 +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 22-------------------- +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:17:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575374 37.634438 37.546535 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.617676 64.679688 64.431641 +util 91.000000 92.000000 89.000000 +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 46.06 seconds +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 12.08 +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 10.15 +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 1.57 +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 23-------------------- +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0060 iter count: 1.0 +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:18:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575496 37.634747 37.546715 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.603027 64.660156 64.431641 +util 94.750000 96.000000 93.000000 +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 46.24 seconds +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 12.18 +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 10.15 +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 1.57 +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 24-------------------- +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:19:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575133 37.633106 37.546768 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.093262 105.126953 105.082031 +util 59.750000 63.000000 58.000000 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 158.14 seconds +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 70.50 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 8.38 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.39 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 25-------------------- +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8027 iter count: 1.0 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8027 iter count: 1.0 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8027 iter count: 1.0 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8027iter count: 1.0 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:22:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575161 37.633232 37.546787 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.793457 64.841797 64.777344 +util 92.750000 94.000000 91.000000 +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 44.64 seconds +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 11.40 +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 10.15 +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 1.57 +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 26-------------------- +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575383 37.633587 37.546810 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.676270 64.724609 64.660156 +util 97.000000 98.000000 95.000000 +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 48.67 seconds +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 13.39 +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 10.14 +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 1.60 +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 27-------------------- +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:23:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575642 37.634579 37.546864 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 65.833496 66.222656 64.666016 +util 99.000000 100.000000 98.000000 +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 49.01 seconds +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 13.56 +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 10.16 +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 1.57 +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 28-------------------- +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:24:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575786 37.635078 37.546864 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656738 64.705078 64.640625 +util 97.000000 98.000000 95.000000 +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 46.18 seconds +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 12.16 +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 10.14 +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 1.58 +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 29-------------------- +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:25:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575386 37.633282 37.546902 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.156738 105.160156 105.146484 +util 66.500000 68.000000 65.000000 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 165.80 seconds +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 74.31 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 8.40 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.40 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 30-------------------- +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8506 iter count: 1.0 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8506 iter count: 1.0 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8506 iter count: 1.0 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8506iter count: 1.0 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:27:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575455 37.633404 37.546963 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.837402 64.841797 64.835938 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 44.61 seconds +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 11.36 +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 10.17 +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 1.57 +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 31-------------------- +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:28:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575515 37.633411 37.546986 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700684 64.822266 64.660156 +util 98.250000 99.000000 96.000000 +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 45.99 seconds +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 12.07 +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 10.14 +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 1.57 +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 32-------------------- +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:29:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575812 37.634438 37.547020 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.778809 64.796875 64.724609 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 43.43 seconds +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 10.78 +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 10.12 +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 1.58 +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 33-------------------- +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:30:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575873 37.634674 37.547047 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.793457 64.841797 64.777344 +util 94.750000 96.000000 93.000000 +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 48.53 seconds +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 13.34 +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 10.15 +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 1.56 +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 34-------------------- +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:31:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575535 37.633327 37.547031 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.156738 105.205078 105.140625 +util 68.000000 69.000000 65.000000 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 160.45 seconds +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 71.63 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 8.42 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.39 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 35-------------------- +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7617 iter count: 1.0 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7617 iter count: 1.0 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7617 iter count: 1.0 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7617iter count: 1.0 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:33:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575542 37.633282 37.547043 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.822754 64.900391 64.796875 +util 95.750000 97.000000 94.000000 +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 47.36 seconds +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 12.75 +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 10.13 +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 1.57 +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 36-------------------- +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:34:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575598 37.633530 37.547031 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.783691 64.861328 64.757812 +util 95.750000 97.000000 94.000000 +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 43.41 seconds +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 10.78 +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 10.14 +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 1.57 +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 37-------------------- +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:35:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576058 37.635174 37.547085 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.793457 64.841797 64.777344 +util 89.250000 90.000000 87.000000 +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 48.72 seconds +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 13.41 +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 10.14 +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 1.59 +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 38-------------------- +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:36:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576062 37.635193 37.547096 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.812988 64.816406 64.802734 +util 90.500000 91.000000 89.000000 +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 46.01 seconds +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 12.08 +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 10.14 +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 1.56 +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 39-------------------- +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:36:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575727 37.633781 37.547119 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.176270 105.199219 105.107422 +util 59.750000 61.000000 58.000000 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 160.61 seconds +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 71.71 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 8.41 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.39 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 40-------------------- +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7852 iter count: 1.0 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7852 iter count: 1.0 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7852 iter count: 1.0 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7852iter count: 1.0 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:39:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575747 37.633850 37.547127 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.876465 64.894531 64.822266 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 49.72 seconds +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 13.92 +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 10.14 +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 1.57 +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 41-------------------- +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:40:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.575865 37.634258 37.547230 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.812988 64.875000 64.626953 +util 97.500000 98.000000 96.000000 +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 46.16 seconds +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 12.14 +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 10.15 +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 1.58 +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 42-------------------- +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:41:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576231 37.635567 37.547356 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.856934 64.894531 64.744141 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 43.44 seconds +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 10.77 +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 10.16 +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 1.57 +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 43-------------------- +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:41:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576309 37.635792 37.547417 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.808105 64.875000 64.607422 +util 98.750000 99.000000 98.000000 +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 45.98 seconds +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 12.06 +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 10.13 +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 1.57 +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 44-------------------- +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:42:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576258 37.635540 37.547424 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.205566 105.341797 105.160156 +util 66.250000 68.000000 64.000000 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 160.77 seconds +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 71.81 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 8.40 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.39 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 45-------------------- +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7744 iter count: 1.0 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7744 iter count: 1.0 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7744 iter count: 1.0 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7744iter count: 1.0 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:45:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576158 37.635040 37.547489 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.836914 64.839844 64.835938 +util 93.750000 95.000000 92.000000 +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 49.76 seconds +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 13.93 +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 10.18 +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 1.57 +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 46-------------------- +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:46:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576159 37.635033 37.547493 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.768555 64.796875 64.683594 +util 95.500000 96.000000 94.000000 +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 46.02 seconds +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 12.08 +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 10.13 +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 1.58 +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 47-------------------- +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:46:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576233 37.635277 37.547516 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.719727 64.781250 64.699219 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 48.68 seconds +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 13.42 +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 10.15 +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 1.57 +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 48-------------------- +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:47:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576243 37.635277 37.547539 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.773438 64.796875 64.703125 +util 90.500000 99.000000 87.000000 +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 48.62 seconds +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 13.38 +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 10.13 +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 1.58 +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 49-------------------- +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:48:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576329 37.635288 37.547611 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.215332 105.238281 105.146484 +util 60.750000 72.000000 57.000000 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 158.01 seconds +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 70.41 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 8.41 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.40 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 50-------------------- +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7695 iter count: 1.0 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7695 iter count: 1.0 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7695 iter count: 1.0 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7695iter count: 1.0 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:51:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576367 37.635296 37.547634 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.837402 64.841797 64.835938 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 47.34 seconds +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 12.76 +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 10.13 +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 1.57 +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 51-------------------- +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:51:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576398 37.635296 37.547756 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.744141 64.820312 64.718750 +util 96.500000 97.000000 95.000000 +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 46.15 seconds +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 12.14 +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 10.16 +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 1.56 +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 52-------------------- +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:52:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576465 37.635540 37.547779 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.807617 64.835938 64.722656 +util 96.250000 100.000000 87.000000 +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 46.10 seconds +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 12.10 +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 10.14 +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 1.58 +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 53-------------------- +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:53:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576478 37.635540 37.547794 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.802734 64.820312 64.796875 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 45.92 seconds +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 12.04 +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 10.15 +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 1.56 +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 54-------------------- +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:54:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576502 37.635555 37.547825 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.132324 105.140625 105.107422 +util 66.000000 67.000000 65.000000 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 160.59 seconds +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 71.70 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 8.41 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.40 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 55-------------------- +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7686 iter count: 1.0 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7686 iter count: 1.0 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7686 iter count: 1.0 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7686iter count: 1.0 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:56:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576567 37.635723 37.547802 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871582 64.914062 64.744141 +util 90.750000 99.000000 88.000000 +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 44.49 seconds +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 11.31 +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 10.15 +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 1.58 +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 56-------------------- +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:57:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576562 37.635723 37.547802 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.837402 64.894531 64.666016 +util 90.500000 99.000000 87.000000 +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 48.59 seconds +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 13.36 +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 10.15 +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 1.58 +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 57-------------------- +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:58:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576571 37.635738 37.547802 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.505371 64.626953 64.464844 +util 89.500000 90.000000 88.000000 +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 46.02 seconds +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 12.07 +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 10.15 +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 1.59 +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 58-------------------- +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:59:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576828 37.636742 37.547802 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.852051 64.953125 64.548828 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 45.94 seconds +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 12.06 +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 10.13 +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 1.56 +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 59-------------------- +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 14:59:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576550 37.635616 37.547798 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.176270 105.224609 105.160156 +util 66.250000 67.000000 65.000000 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 160.56 seconds +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 71.70 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 8.39 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.39 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 60-------------------- +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7422 iter count: 1.0 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7422 iter count: 1.0 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7422 iter count: 1.0 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7422iter count: 1.0 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:02:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576710 37.636265 37.547836 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.905762 64.914062 64.880859 +util 98.250000 99.000000 97.000000 +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 49.81 seconds +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 13.98 +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 10.13 +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 1.56 +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 61-------------------- +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:03:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576761 37.636482 37.547821 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.910645 64.933594 64.841797 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 46.03 seconds +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 12.07 +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 10.15 +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 1.59 +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 62-------------------- +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:04:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576777 37.636505 37.547859 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.881348 64.933594 64.724609 +util 91.500000 92.000000 90.000000 +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 43.29 seconds +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 10.72 +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 10.15 +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 1.57 +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 63-------------------- +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:04:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576816 37.636532 37.547867 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871582 64.894531 64.802734 +util 90.750000 91.000000 90.000000 +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 45.99 seconds +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 12.08 +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 10.13 +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 1.57 +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 64-------------------- +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:05:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576641 37.635777 37.547909 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.249512 105.341797 105.218750 +util 67.500000 75.000000 58.000000 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 163.36 seconds +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 73.06 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 8.41 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.38 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 65-------------------- +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7061 iter count: 1.0 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7061 iter count: 1.0 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7061 iter count: 1.0 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7061iter count: 1.0 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:08:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576652 37.635777 37.547909 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.641602 64.878906 64.562500 +util 98.250000 99.000000 97.000000 +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 47.32 seconds +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 12.72 +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 10.15 +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 1.59 +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 66-------------------- +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:09:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576730 37.635979 37.548008 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.578125 64.800781 64.503906 +util 98.750000 100.000000 97.000000 +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 46.01 seconds +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 12.06 +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 10.16 +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 1.57 +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 67-------------------- +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:09:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576756 37.635979 37.548016 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.602539 64.839844 64.523438 +util 98.750000 99.000000 98.000000 +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 48.61 seconds +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 13.34 +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 10.17 +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 1.58 +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 68-------------------- +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:10:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576781 37.635979 37.548016 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.573242 64.781250 64.503906 +util 90.500000 91.000000 89.000000 +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 48.69 seconds +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 13.39 +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 10.15 +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 1.57 +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 69-------------------- +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:11:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576662 37.635468 37.548016 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.107910 105.126953 105.101562 +util 68.250000 73.000000 56.000000 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 160.57 seconds +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 71.72 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 8.38 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.39 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 70-------------------- +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7305 iter count: 1.0 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7305 iter count: 1.0 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7305 iter count: 1.0 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7305iter count: 1.0 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:14:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576675 37.635471 37.548031 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.875977 64.937500 64.855469 +util 96.250000 100.000000 87.000000 +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 47.19 seconds +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 12.65 +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 10.15 +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 1.58 +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 71-------------------- +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:15:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576806 37.635952 37.548050 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.670898 64.757812 64.410156 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 45.96 seconds +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 12.04 +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 10.14 +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 1.58 +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 72-------------------- +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:15:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576808 37.635967 37.548065 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.783203 64.859375 64.757812 +util 93.500000 94.000000 92.000000 +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 48.52 seconds +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 13.35 +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 10.13 +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 1.57 +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 73-------------------- +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:16:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576816 37.635952 37.548065 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.802734 64.878906 64.777344 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 48.49 seconds +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 13.32 +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 10.14 +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 1.56 +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 74-------------------- +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:17:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576737 37.635563 37.548134 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.132324 105.224609 105.101562 +util 63.500000 64.000000 62.000000 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 160.33 seconds +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 71.59 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 8.38 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.40 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 75-------------------- +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6758 iter count: 1.0 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6758 iter count: 1.0 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6758 iter count: 1.0 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6758iter count: 1.0 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:20:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576752 37.635540 37.548183 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.842285 64.855469 64.802734 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 49.67 seconds +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 13.92 +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 10.12 +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 1.57 +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 76-------------------- +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:20:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577020 37.636559 37.548214 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.798340 64.802734 64.796875 +util 91.000000 100.000000 88.000000 +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 48.64 seconds +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 13.39 +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 10.15 +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 1.57 +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 77-------------------- +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:21:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577026 37.636559 37.548218 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.715332 64.763672 64.699219 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 48.55 seconds +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 13.34 +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 10.16 +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 1.57 +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 78-------------------- +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:22:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577041 37.636562 37.548237 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.720215 64.783203 64.699219 +util 92.500000 93.000000 91.000000 +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 48.50 seconds +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 13.32 +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 10.14 +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 1.57 +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 79-------------------- +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:23:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576715 37.635231 37.548256 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.166504 105.244141 105.140625 +util 60.250000 61.000000 59.000000 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 155.39 seconds +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 69.10 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 8.40 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.39 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 80-------------------- +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7197 iter count: 1.0 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7197 iter count: 1.0 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7197 iter count: 1.0 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7197iter count: 1.0 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:25:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576714 37.635235 37.548264 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.837402 64.900391 64.816406 +util 91.500000 92.000000 90.000000 +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 44.61 seconds +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 11.38 +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 10.15 +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 1.56 +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 81-------------------- +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576839 37.635735 37.548267 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.764160 64.841797 64.738281 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 43.52 seconds +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 10.72 +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 10.15 +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 1.57 +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 82-------------------- +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:27:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576839 37.635735 37.548267 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.754395 64.861328 64.718750 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 46.01 seconds +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 12.06 +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 10.17 +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 1.56 +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 83-------------------- +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:28:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576852 37.635735 37.548267 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817871 64.880859 64.796875 +util 93.500000 94.000000 92.000000 +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 46.09 seconds +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 12.11 +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 10.14 +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 1.58 +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 84-------------------- +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:28:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576777 37.635204 37.548378 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.220215 105.238281 105.166016 +util 60.250000 61.000000 59.000000 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 160.37 seconds +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 71.60 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 8.42 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.40 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 85-------------------- +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6826 iter count: 1.0 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6826 iter count: 1.0 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6826 iter count: 1.0 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6826iter count: 1.0 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576780 37.635204 37.548389 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.856445 64.917969 64.835938 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 47.11 seconds +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 12.62 +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 10.15 +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 1.58 +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 86-------------------- +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:32:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576842 37.635456 37.548397 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.822266 64.898438 64.796875 +util 96.250000 100.000000 87.000000 +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 46.03 seconds +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 12.08 +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 10.15 +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 1.56 +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 87-------------------- +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:33:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577034 37.636230 37.548389 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.827148 64.917969 64.796875 +util 91.500000 92.000000 90.000000 +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 46.07 seconds +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 12.10 +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 10.15 +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 1.57 +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 88-------------------- +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:33:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577028 37.636208 37.548389 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.841797 64.917969 64.816406 +util 93.250000 94.000000 92.000000 +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 46.02 seconds +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 12.08 +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 10.14 +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 1.56 +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 89-------------------- +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:34:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576735 37.634995 37.548397 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.137207 105.140625 105.126953 +util 68.000000 73.000000 55.000000 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 163.13 seconds +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 72.97 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 8.41 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.40 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 90-------------------- +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7168 iter count: 1.0 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7168 iter count: 1.0 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7168 iter count: 1.0 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7168iter count: 1.0 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:37:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576730 37.634983 37.548401 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.835938 64.761719 +util 91.000000 100.000000 88.000000 +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 44.43 seconds +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 11.28 +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 10.14 +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 1.60 +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 91-------------------- +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:38:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576754 37.635017 37.548401 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.739258 64.757812 64.683594 +util 96.750000 97.000000 96.000000 +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 48.68 seconds +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 13.41 +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 10.15 +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 1.57 +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 92-------------------- +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:39:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577080 37.636219 37.548397 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.758789 64.777344 64.703125 +util 96.500000 100.000000 88.000000 +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 45.87 seconds +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 12.02 +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 10.16 +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 1.55 +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 93-------------------- +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:39:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577085 37.636242 37.548397 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.763672 64.796875 64.664062 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 43.59 seconds +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 10.86 +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 10.13 +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 1.57 +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 94-------------------- +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:40:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577226 37.636765 37.548412 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.190918 105.199219 105.166016 +util 66.500000 67.000000 65.000000 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 158.15 seconds +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 70.48 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 8.44 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.39 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 95-------------------- +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7500 iter count: 1.0 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7500 iter count: 1.0 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7500 iter count: 1.0 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7500iter count: 1.0 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:43:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576999 37.635872 37.548405 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.798340 64.861328 64.777344 +util 92.750000 93.000000 92.000000 +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 44.55 seconds +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 11.33 +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 10.14 +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 1.59 +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 96-------------------- +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:43:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.577001 37.635860 37.548424 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.695801 64.861328 64.640625 +util 89.000000 90.000000 88.000000 +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 46.14 seconds +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.14 +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 10.15 +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 1.57 +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 97-------------------- +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:44:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576999 37.635872 37.548405 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.686035 64.705078 64.679688 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 45.95 seconds +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.03 +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 10.15 +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 1.57 +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 98-------------------- +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:45:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576999 37.635872 37.548405 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.705566 64.783203 64.679688 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 45.99 seconds +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 12.07 +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 10.14 +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 1.57 +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 99-------------------- +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:46:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.576996 37.635860 37.548405 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.151855 105.244141 105.121094 +util 68.250000 69.000000 68.000000 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 165.58 seconds +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 74.19 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 8.39 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.40 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 100-------------------- +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7168 iter count: 1.0 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7168 iter count: 1.0 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7168 iter count: 1.0 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7168iter count: 1.0 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 15:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 15:49:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 18.01 seconds +[May 4, 2026 - 15:49:14 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[May 4, 2026 - 15:49:14 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000100 +[May 4, 2026 - 15:49:32 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 100 +[May 4, 2026 - 15:49:32 | INFO | fastgen.trainer:validate:358 ] Validation iteration 100 +[May 4, 2026 - 15:49:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:49:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 4, 2026 - 15:49:38 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 4, 2026 - 15:49:39 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.575 +[May 4, 2026 - 15:49:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:49:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 15:49:44 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.973 +[May 4, 2026 - 15:49:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:49:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 15:49:49 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.233 +[May 4, 2026 - 15:49:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:49:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 15:49:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.285 +[May 4, 2026 - 15:49:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:49:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 15:49:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=5.359 +[May 4, 2026 - 15:49:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:50:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 15:50:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.753 +[May 4, 2026 - 15:50:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:50:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 15:50:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.144 +[May 4, 2026 - 15:50:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:50:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 15:50:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.008 +[May 4, 2026 - 15:50:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:50:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 15:50:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.479 +[May 4, 2026 - 15:50:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 15:50:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 15:50:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.503 +[May 4, 2026 - 15:50:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 100-------------------- +[May 4, 2026 - 15:50:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 15:50:40 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 100 +[May 4, 2026 - 15:50:40 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.331 (n=10) +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415874 41.501461 41.378300 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.207031 68.214844 68.183594 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 101 : avg iteration time 146.22 seconds +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 101 : data loading time 0.00 +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 101 : avg forward pass time 10.37 +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 101 : backward pass time 10.13 +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 101 : optimizer step time 1.58 +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 101-------------------- +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:51:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415664 41.500519 41.378345 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.133789 68.175781 68.007812 +util 97.500000 98.000000 97.000000 +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 102 : avg iteration time 48.96 seconds +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 102 : data loading time 0.00 +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 102 : avg forward pass time 13.52 +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 102 : backward pass time 10.19 +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 102 : optimizer step time 1.56 +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 102-------------------- +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:52:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415709 41.500706 41.378345 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.133789 68.175781 68.007812 +util 91.250000 92.000000 90.000000 +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 103 : avg iteration time 43.53 seconds +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 103 : data loading time 0.00 +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 103 : avg forward pass time 10.83 +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 103 : backward pass time 10.13 +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 103 : optimizer step time 1.56 +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 103-------------------- +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:52:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415138 41.498447 41.378300 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.133789 68.175781 68.007812 +util 92.000000 93.000000 91.000000 +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 104 : avg iteration time 46.21 seconds +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 104 : data loading time 0.00 +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 104 : avg forward pass time 12.15 +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 104 : backward pass time 10.15 +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 104 : optimizer step time 1.58 +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 104-------------------- +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:53:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416135 41.498699 41.379456 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.726562 107.544922 +util 65.500000 67.000000 65.000000 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 105 : avg iteration time 161.27 seconds +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 105 : data loading time 0.00 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 105 : avg forward pass time 72.04 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 105 : backward pass time 8.42 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 105 : optimizer step time 0.41 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 105-------------------- +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7148 iter count: 1.0 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7148 iter count: 1.0 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7148 iter count: 1.0 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7148iter count: 1.0 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:56:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416186 41.498676 41.379513 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.281250 69.894531 67.441406 +util 91.250000 99.000000 88.000000 +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 106 : avg iteration time 44.68 seconds +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 106 : data loading time 0.00 +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 106 : avg forward pass time 11.40 +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 106 : backward pass time 10.15 +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 106 : optimizer step time 1.58 +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 106-------------------- +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:57:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416308 41.499157 41.379501 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.320312 69.992188 67.304688 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 107 : avg iteration time 48.97 seconds +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 107 : data loading time 0.00 +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 107 : avg forward pass time 13.55 +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 107 : backward pass time 10.14 +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 107 : optimizer step time 1.56 +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 107-------------------- +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416332 41.499157 41.379532 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.393555 70.070312 67.363281 +util 92.000000 93.000000 91.000000 +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 108 : avg iteration time 48.87 seconds +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 108 : data loading time 0.00 +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 108 : avg forward pass time 13.52 +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 108 : backward pass time 10.13 +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 108 : optimizer step time 1.57 +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 108-------------------- +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:58:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416390 41.499195 41.379555 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 69.359375 70.031250 67.343750 +util 96.750000 97.000000 96.000000 +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 109 : avg iteration time 46.06 seconds +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 109 : data loading time 0.00 +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 109 : avg forward pass time 12.09 +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 109 : backward pass time 10.14 +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 109 : optimizer step time 1.58 +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 109-------------------- +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 15:59:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416023 41.497730 41.379509 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.609375 107.564453 +util 63.750000 64.000000 63.000000 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 110 : avg iteration time 160.99 seconds +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 110 : data loading time 0.00 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 110 : avg forward pass time 71.92 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 110 : backward pass time 8.40 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 110 : optimizer step time 0.39 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 110-------------------- +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7188 iter count: 1.0 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7188 iter count: 1.0 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7188 iter count: 1.0 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7188iter count: 1.0 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:02:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416026 41.497730 41.379539 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.104980 67.285156 67.044922 +util 94.500000 95.000000 94.000000 +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 111 : avg iteration time 47.39 seconds +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 111 : data loading time 0.00 +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 111 : avg forward pass time 12.77 +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 111 : backward pass time 10.16 +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 111 : optimizer step time 1.56 +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 111-------------------- +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:02:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416224 41.498482 41.379536 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.314941 67.318359 67.304688 +util 99.750000 100.000000 99.000000 +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 112 : avg iteration time 46.08 seconds +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 112 : data loading time 0.00 +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 112 : avg forward pass time 12.10 +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 112 : backward pass time 10.15 +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 112 : optimizer step time 1.56 +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 112-------------------- +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:03:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416324 41.498741 41.379559 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.314941 67.318359 67.304688 +util 95.500000 96.000000 95.000000 +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 113 : avg iteration time 43.54 seconds +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 113 : data loading time 0.00 +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 113 : avg forward pass time 10.83 +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 113 : backward pass time 10.13 +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 113 : optimizer step time 1.58 +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 113-------------------- +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:04:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416325 41.498756 41.379593 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.324707 67.343750 67.318359 +util 97.750000 99.000000 97.000000 +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 114 : avg iteration time 43.42 seconds +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 114 : data loading time 0.00 +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 114 : avg forward pass time 10.79 +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 114 : backward pass time 10.14 +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 114 : optimizer step time 1.56 +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 114-------------------- +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:05:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416098 41.497692 41.379608 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.785156 107.505859 +util 68.750000 74.000000 56.000000 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 115 : avg iteration time 163.49 seconds +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 115 : data loading time 0.00 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 115 : avg forward pass time 73.16 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 115 : backward pass time 8.40 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 115 : optimizer step time 0.40 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 115-------------------- +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6982 iter count: 1.0 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6982 iter count: 1.0 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6982 iter count: 1.0 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6982iter count: 1.0 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:07:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416104 41.497692 41.379631 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.127930 67.382812 67.042969 +util 96.000000 97.000000 95.000000 +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 116 : avg iteration time 49.91 seconds +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 116 : data loading time 0.00 +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 116 : avg forward pass time 14.01 +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 116 : backward pass time 10.14 +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 116 : optimizer step time 1.59 +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 116-------------------- +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:08:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416282 41.498432 41.379612 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.289062 67.316406 67.207031 +util 90.250000 91.000000 89.000000 +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 117 : avg iteration time 43.52 seconds +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 117 : data loading time 0.00 +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 117 : avg forward pass time 10.83 +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 117 : backward pass time 10.16 +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 117 : optimizer step time 1.57 +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 117-------------------- +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:09:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416425 41.498947 41.379639 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.083984 67.207031 67.042969 +util 90.500000 91.000000 90.000000 +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 118 : avg iteration time 43.41 seconds +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 118 : data loading time 0.00 +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 118 : avg forward pass time 10.76 +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 118 : backward pass time 10.16 +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 118 : optimizer step time 1.58 +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 118-------------------- +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:10:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416195 41.497967 41.379734 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.333008 67.375000 67.207031 +util 99.000000 100.000000 98.000000 +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 119 : avg iteration time 46.11 seconds +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 119 : data loading time 0.00 +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 119 : avg forward pass time 12.12 +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 119 : backward pass time 10.14 +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 119 : optimizer step time 1.58 +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 119-------------------- +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:10:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415751 41.496071 41.379852 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.623047 107.609375 +util 69.250000 70.000000 68.000000 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 120 : avg iteration time 163.21 seconds +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 120 : data loading time 0.00 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 120 : avg forward pass time 73.02 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 120 : backward pass time 8.42 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 120 : optimizer step time 0.39 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 120-------------------- +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.7383 iter count: 1.0 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7383 iter count: 1.0 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.7383 iter count: 1.0 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7383iter count: 1.0 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:13:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415787 41.496071 41.379852 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.113281 67.382812 67.023438 +util 95.250000 96.000000 94.000000 +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 121 : avg iteration time 44.55 seconds +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 121 : data loading time 0.00 +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 121 : avg forward pass time 11.34 +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 121 : backward pass time 10.14 +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 121 : optimizer step time 1.59 +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 121-------------------- +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:14:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415937 41.496571 41.379898 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.162109 67.167969 67.160156 +util 92.250000 93.000000 91.000000 +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 122 : avg iteration time 46.16 seconds +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 122 : data loading time 0.00 +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 122 : avg forward pass time 12.12 +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 122 : backward pass time 10.18 +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 122 : optimizer step time 1.57 +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 122-------------------- +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:15:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416178 41.497570 41.379883 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.157227 67.160156 67.148438 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 123 : avg iteration time 46.31 seconds +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 123 : data loading time 0.00 +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 123 : avg forward pass time 12.24 +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 123 : backward pass time 10.13 +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 123 : optimizer step time 1.57 +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 123-------------------- +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:15:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416176 41.497570 41.379875 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.201172 67.324219 67.160156 +util 92.000000 93.000000 91.000000 +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 124 : avg iteration time 45.89 seconds +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 124 : data loading time 0.00 +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 124 : avg forward pass time 12.01 +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 124 : backward pass time 10.15 +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 124 : optimizer step time 1.57 +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 124-------------------- +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:16:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416199 41.497570 41.379879 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.648438 107.642578 +util 63.250000 64.000000 62.000000 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 125 : avg iteration time 158.24 seconds +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 125 : data loading time 0.00 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 125 : avg forward pass time 70.55 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 125 : backward pass time 8.38 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 125 : optimizer step time 0.40 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 125-------------------- +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6172 iter count: 1.0 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6172 iter count: 1.0 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6172 iter count: 1.0 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6172iter count: 1.0 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:19:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415984 41.496731 41.379879 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.310059 67.357422 67.167969 +util 91.500000 92.000000 91.000000 +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 126 : avg iteration time 49.76 seconds +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 126 : data loading time 0.00 +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 126 : avg forward pass time 13.94 +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 126 : backward pass time 10.15 +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 126 : optimizer step time 1.57 +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 126-------------------- +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:20:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.415983 41.496716 41.379879 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.192871 67.201172 67.167969 +util 91.750000 93.000000 91.000000 +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 127 : avg iteration time 43.40 seconds +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 127 : data loading time 0.00 +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 127 : avg forward pass time 10.77 +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 127 : backward pass time 10.15 +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 127 : optimizer step time 1.57 +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 127-------------------- +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:20:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416778 41.499866 41.379879 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.324707 67.376953 67.167969 +util 95.500000 96.000000 95.000000 +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 128 : avg iteration time 43.46 seconds +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 128 : data loading time 0.00 +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 128 : avg forward pass time 10.79 +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 128 : backward pass time 10.15 +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 128 : optimizer step time 1.59 +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 128-------------------- +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:21:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416773 41.499851 41.379879 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.046387 67.167969 67.005859 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 129 : avg iteration time 43.44 seconds +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 129 : data loading time 0.00 +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 129 : avg forward pass time 10.79 +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 129 : backward pass time 10.14 +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 129 : optimizer step time 1.58 +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 129-------------------- +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:22:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416810 41.499973 41.379898 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.648438 107.623047 +util 62.500000 63.000000 62.000000 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 130 : avg iteration time 161.12 seconds +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 130 : data loading time 0.00 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 130 : avg forward pass time 71.96 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 130 : backward pass time 8.41 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 130 : optimizer step time 0.39 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 130-------------------- +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6689 iter count: 1.0 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6689 iter count: 1.0 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6689 iter count: 1.0 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6689iter count: 1.0 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:25:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416714 41.499496 41.379913 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.119629 67.343750 67.044922 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 131 : avg iteration time 49.69 seconds +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 131 : data loading time 0.00 +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 131 : avg forward pass time 13.92 +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 131 : backward pass time 10.14 +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 131 : optimizer step time 1.58 +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 131-------------------- +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:25:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416729 41.499542 41.379925 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.227051 67.363281 67.181641 +util 88.000000 89.000000 87.000000 +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 132 : avg iteration time 46.05 seconds +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 132 : data loading time 0.00 +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 132 : avg forward pass time 12.08 +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 132 : backward pass time 10.18 +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 132 : optimizer step time 1.56 +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 132-------------------- +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:26:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416742 41.499607 41.379913 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.246582 67.382812 67.201172 +util 89.750000 91.000000 89.000000 +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 133 : avg iteration time 46.02 seconds +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 133 : data loading time 0.00 +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 133 : avg forward pass time 12.06 +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 133 : backward pass time 10.15 +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 133 : optimizer step time 1.58 +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 133-------------------- +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:27:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416737 41.499584 41.379913 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.236816 67.343750 67.201172 +util 89.750000 91.000000 89.000000 +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 134 : avg iteration time 48.80 seconds +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 134 : data loading time 0.00 +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 134 : avg forward pass time 13.47 +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 134 : backward pass time 10.14 +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 134 : optimizer step time 1.57 +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 134-------------------- +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:28:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416779 41.499622 41.379913 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.526855 107.648438 107.486328 +util 69.750000 70.000000 69.000000 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 135 : avg iteration time 158.34 seconds +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 135 : data loading time 0.00 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 135 : avg forward pass time 70.58 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 135 : backward pass time 8.41 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 135 : optimizer step time 0.40 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 135-------------------- +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6748 iter count: 1.0 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6748 iter count: 1.0 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6748 iter count: 1.0 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6748iter count: 1.0 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:30:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416754 41.499546 41.379913 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.666504 67.767578 67.363281 +util 95.250000 96.000000 94.000000 +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 136 : avg iteration time 47.13 seconds +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 136 : data loading time 0.00 +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 136 : avg forward pass time 12.63 +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 136 : backward pass time 10.15 +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 136 : optimizer step time 1.57 +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 136-------------------- +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:31:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416763 41.499561 41.379913 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.593262 67.669922 67.363281 +util 88.250000 89.000000 87.000000 +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 137 : avg iteration time 46.12 seconds +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 137 : data loading time 0.00 +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 137 : avg forward pass time 12.13 +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 137 : backward pass time 10.14 +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 137 : optimizer step time 1.58 +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 137-------------------- +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:32:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416754 41.499546 41.379913 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.593262 67.669922 67.363281 +util 93.000000 94.000000 92.000000 +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 138 : avg iteration time 46.05 seconds +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 138 : data loading time 0.00 +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 138 : avg forward pass time 12.10 +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 138 : backward pass time 10.13 +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 138 : optimizer step time 1.57 +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 138-------------------- +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:33:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416791 41.499649 41.379929 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.593262 67.669922 67.363281 +util 99.000000 100.000000 98.000000 +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 139 : avg iteration time 46.08 seconds +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 139 : data loading time 0.00 +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 139 : avg forward pass time 12.08 +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 139 : backward pass time 10.16 +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 139 : optimizer step time 1.59 +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 139-------------------- +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:33:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416670 41.499180 41.379936 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.561035 107.609375 107.544922 +util 68.750000 74.000000 56.000000 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 140 : avg iteration time 163.26 seconds +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 140 : data loading time 0.00 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 140 : avg forward pass time 73.03 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 140 : backward pass time 8.40 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 140 : optimizer step time 0.40 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 140-------------------- +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6914 iter count: 1.0 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6914 iter count: 1.0 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6914 iter count: 1.0 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6914iter count: 1.0 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:36:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416686 41.499199 41.379936 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.490723 67.552734 67.304688 +util 96.000000 100.000000 86.000000 +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 141 : avg iteration time 47.38 seconds +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 141 : data loading time 0.00 +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 141 : avg forward pass time 12.77 +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 141 : backward pass time 10.14 +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 141 : optimizer step time 1.57 +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 141-------------------- +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:37:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416691 41.499191 41.379936 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.642090 67.748047 67.324219 +util 97.500000 98.000000 97.000000 +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 142 : avg iteration time 45.93 seconds +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 142 : data loading time 0.00 +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 142 : avg forward pass time 12.06 +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 142 : backward pass time 10.14 +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 142 : optimizer step time 1.56 +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 142-------------------- +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:38:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416695 41.499184 41.379959 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.461426 67.773438 67.357422 +util 88.500000 90.000000 87.000000 +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 143 : avg iteration time 46.10 seconds +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 143 : data loading time 0.00 +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 143 : avg forward pass time 12.12 +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 143 : backward pass time 10.15 +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 143 : optimizer step time 1.56 +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 143-------------------- +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:39:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416767 41.499477 41.379936 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.510254 67.630859 67.148438 +util 91.750000 93.000000 91.000000 +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 144 : avg iteration time 43.43 seconds +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 144 : data loading time 0.00 +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 144 : avg forward pass time 10.80 +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 144 : backward pass time 10.13 +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 144 : optimizer step time 1.57 +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 144-------------------- +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:39:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416524 41.498478 41.379948 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.667969 107.603516 +util 63.000000 64.000000 62.000000 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 145 : avg iteration time 160.63 seconds +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 145 : data loading time 0.00 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 145 : avg forward pass time 71.73 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 145 : backward pass time 8.41 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 145 : optimizer step time 0.39 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 145-------------------- +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6875 iter count: 1.0 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6875 iter count: 1.0 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6875 iter count: 1.0 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6875iter count: 1.0 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:42:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416533 41.498474 41.379971 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.596680 67.707031 67.265625 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 146 : avg iteration time 47.27 seconds +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 146 : data loading time 0.00 +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 146 : avg forward pass time 12.71 +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 146 : backward pass time 10.14 +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 146 : optimizer step time 1.57 +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 146-------------------- +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:43:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416533 41.498478 41.379978 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.284180 67.421875 67.238281 +util 98.750000 100.000000 98.000000 +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 147 : avg iteration time 48.78 seconds +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 147 : data loading time 0.00 +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 147 : avg forward pass time 13.46 +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 147 : backward pass time 10.15 +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 147 : optimizer step time 1.57 +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 147-------------------- +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:44:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416544 41.498474 41.379982 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.293945 67.402344 67.257812 +util 89.750000 91.000000 89.000000 +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 148 : avg iteration time 46.30 seconds +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 148 : data loading time 0.00 +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 148 : avg forward pass time 12.23 +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 148 : backward pass time 10.14 +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 148 : optimizer step time 1.57 +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 148-------------------- +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:44:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416538 41.498474 41.379982 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.402344 67.375000 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 149 : avg iteration time 43.72 seconds +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 149 : data loading time 0.00 +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 149 : avg forward pass time 10.93 +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 149 : backward pass time 10.15 +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 149 : optimizer step time 1.58 +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 149-------------------- +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:45:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416444 41.497971 41.379971 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.668457 107.746094 107.642578 +util 65.250000 66.000000 65.000000 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 150 : avg iteration time 161.55 seconds +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 150 : data loading time 0.00 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 150 : avg forward pass time 72.18 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 150 : backward pass time 8.41 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 150 : optimizer step time 0.39 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 150-------------------- +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6631 iter count: 1.0 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6631 iter count: 1.0 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6631 iter count: 1.0 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6631iter count: 1.0 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:48:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416444 41.497971 41.379993 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.637207 67.708984 67.421875 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 151 : avg iteration time 47.26 seconds +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 151 : data loading time 0.00 +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 151 : avg forward pass time 12.70 +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 151 : backward pass time 10.14 +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 151 : optimizer step time 1.58 +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 151-------------------- +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:49:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416460 41.498055 41.379971 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729980 67.851562 67.689453 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 152 : avg iteration time 48.79 seconds +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 152 : data loading time 0.00 +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 152 : avg forward pass time 13.45 +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 152 : backward pass time 10.14 +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 152 : optimizer step time 1.58 +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 152-------------------- +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:49:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416520 41.498299 41.379993 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.578613 67.708984 67.187500 +util 94.750000 96.000000 94.000000 +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 153 : avg iteration time 46.13 seconds +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 153 : data loading time 0.00 +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 153 : avg forward pass time 12.12 +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 153 : backward pass time 10.15 +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 153 : optimizer step time 1.58 +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 153-------------------- +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:50:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416528 41.498299 41.379971 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.495605 67.533203 67.382812 +util 88.500000 89.000000 88.000000 +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 154 : avg iteration time 43.52 seconds +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 154 : data loading time 0.00 +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 154 : avg forward pass time 10.83 +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 154 : backward pass time 10.14 +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 154 : optimizer step time 1.58 +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 154-------------------- +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:51:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416286 41.497349 41.379978 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.707031 107.583984 +util 68.250000 73.000000 56.000000 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 155 : avg iteration time 158.58 seconds +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 155 : data loading time 0.00 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 155 : avg forward pass time 70.68 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 155 : backward pass time 8.40 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 155 : optimizer step time 0.39 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 155-------------------- +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6260 iter count: 1.0 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6260 iter count: 1.0 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6260 iter count: 1.0 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6260iter count: 1.0 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:53:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416369 41.497570 41.380001 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.212402 67.246094 67.201172 +util 94.000000 100.000000 88.000000 +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 156 : avg iteration time 50.40 seconds +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 156 : data loading time 0.00 +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 156 : avg forward pass time 14.27 +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 156 : backward pass time 10.14 +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 156 : optimizer step time 1.56 +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 156-------------------- +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:54:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416512 41.498100 41.379978 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.051270 67.246094 66.986328 +util 96.500000 100.000000 88.000000 +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 157 : avg iteration time 46.39 seconds +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 157 : data loading time 0.00 +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 157 : avg forward pass time 12.22 +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 157 : backward pass time 10.21 +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 157 : optimizer step time 1.57 +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 157-------------------- +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:55:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416508 41.498100 41.379986 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.207520 67.460938 67.123047 +util 91.500000 92.000000 91.000000 +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 158 : avg iteration time 46.07 seconds +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 158 : data loading time 0.00 +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 158 : avg forward pass time 12.10 +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 158 : backward pass time 10.13 +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 158 : optimizer step time 1.59 +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 158-------------------- +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:56:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416584 41.498295 41.380001 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.285645 67.421875 67.240234 +util 97.750000 99.000000 97.000000 +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 159 : avg iteration time 46.07 seconds +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 159 : data loading time 0.00 +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 159 : avg forward pass time 12.09 +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 159 : backward pass time 10.14 +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 159 : optimizer step time 1.58 +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 159-------------------- +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:57:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416275 41.497040 41.380001 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.628906 107.564453 +util 63.500000 69.000000 51.000000 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 160 : avg iteration time 163.86 seconds +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 160 : data loading time 0.00 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 160 : avg forward pass time 73.31 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 160 : backward pass time 8.38 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 160 : optimizer step time 0.44 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 160-------------------- +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6641 iter count: 1.0 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6641 iter count: 1.0 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6641 iter count: 1.0 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6641iter count: 1.0 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 16:59:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416309 41.497017 41.380016 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.235352 67.343750 67.199219 +util 94.250000 95.000000 93.000000 +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 161 : avg iteration time 44.69 seconds +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 161 : data loading time 0.00 +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 161 : avg forward pass time 11.42 +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 161 : backward pass time 10.13 +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 161 : optimizer step time 1.58 +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 161-------------------- +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416424 41.497471 41.380016 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.123047 67.363281 67.042969 +util 98.750000 100.000000 98.000000 +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 162 : avg iteration time 46.26 seconds +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 162 : data loading time 0.00 +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 162 : avg forward pass time 12.19 +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 162 : backward pass time 10.15 +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 162 : optimizer step time 1.57 +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 162-------------------- +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416670 41.498440 41.380016 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.201172 67.207031 67.199219 +util 98.750000 99.000000 98.000000 +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 163 : avg iteration time 46.20 seconds +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 163 : data loading time 0.00 +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 163 : avg forward pass time 12.18 +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 163 : backward pass time 10.12 +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 163 : optimizer step time 1.58 +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 163-------------------- +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416736 41.498680 41.380039 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.250000 67.460938 67.179688 +util 97.750000 99.000000 97.000000 +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 164 : avg iteration time 46.39 seconds +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 164 : data loading time 0.00 +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 164 : avg forward pass time 12.26 +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 164 : backward pass time 10.14 +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 164 : optimizer step time 1.57 +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 164-------------------- +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:02:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416249 41.496742 41.380020 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.628906 107.544922 +util 70.750000 72.000000 70.000000 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 165 : avg iteration time 160.85 seconds +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 165 : data loading time 0.00 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 165 : avg forward pass time 71.84 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 165 : backward pass time 8.39 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 165 : optimizer step time 0.40 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 165-------------------- +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6670 iter count: 1.0 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6670 iter count: 1.0 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6670 iter count: 1.0 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6670iter count: 1.0 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:05:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416250 41.496742 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.466309 67.494141 67.382812 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 166 : avg iteration time 49.99 seconds +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 166 : data loading time 0.00 +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 166 : avg forward pass time 14.06 +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 166 : backward pass time 10.16 +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 166 : optimizer step time 1.58 +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 166-------------------- +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:06:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416479 41.497658 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.368652 67.402344 67.357422 +util 92.000000 93.000000 91.000000 +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 167 : avg iteration time 48.89 seconds +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 167 : data loading time 0.00 +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 167 : avg forward pass time 13.47 +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 167 : backward pass time 10.22 +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 167 : optimizer step time 1.59 +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 167-------------------- +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:07:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416692 41.498409 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.474609 67.363281 +util 93.250000 100.000000 87.000000 +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 168 : avg iteration time 46.34 seconds +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 168 : data loading time 0.00 +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 168 : avg forward pass time 12.23 +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 168 : backward pass time 10.14 +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 168 : optimizer step time 1.58 +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 168-------------------- +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:08:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416722 41.498489 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.559082 67.669922 67.226562 +util 93.500000 100.000000 87.000000 +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 169 : avg iteration time 46.20 seconds +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 169 : data loading time 0.00 +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 169 : avg forward pass time 12.13 +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 169 : backward pass time 10.14 +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 169 : optimizer step time 1.58 +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 169-------------------- +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:08:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416418 41.497269 41.380020 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.526855 107.648438 107.486328 +util 60.000000 61.000000 59.000000 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 170 : avg iteration time 161.26 seconds +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 170 : data loading time 0.00 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 170 : avg forward pass time 72.00 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 170 : backward pass time 8.40 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 170 : optimizer step time 0.40 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 170-------------------- +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6621 iter count: 1.0 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6621 iter count: 1.0 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6621 iter count: 1.0 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6621iter count: 1.0 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:11:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416429 41.497314 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.344238 67.363281 67.337891 +util 98.250000 99.000000 97.000000 +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 171 : avg iteration time 47.48 seconds +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 171 : data loading time 0.00 +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 171 : avg forward pass time 12.82 +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 171 : backward pass time 10.13 +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 171 : optimizer step time 1.59 +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 171-------------------- +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:12:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416555 41.497822 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.363770 67.382812 67.357422 +util 93.500000 100.000000 87.000000 +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 172 : avg iteration time 46.06 seconds +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 172 : data loading time 0.00 +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 172 : avg forward pass time 12.10 +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 172 : backward pass time 10.15 +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 172 : optimizer step time 1.58 +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 172-------------------- +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:13:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416815 41.498852 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.241699 67.304688 67.220703 +util 94.750000 96.000000 94.000000 +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 173 : avg iteration time 46.29 seconds +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 173 : data loading time 0.00 +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 173 : avg forward pass time 12.20 +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 173 : backward pass time 10.16 +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 173 : optimizer step time 1.57 +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 173-------------------- +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:13:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416833 41.498806 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.354004 67.402344 67.337891 +util 98.250000 99.000000 97.000000 +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 174 : avg iteration time 46.45 seconds +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 174 : data loading time 0.00 +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 174 : avg forward pass time 12.28 +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 174 : backward pass time 10.15 +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 174 : optimizer step time 1.57 +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 174-------------------- +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:14:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416483 41.497360 41.380020 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.551270 107.570312 107.544922 +util 65.250000 74.000000 57.000000 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 175 : avg iteration time 158.48 seconds +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 175 : data loading time 0.00 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 175 : avg forward pass time 70.62 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 175 : backward pass time 8.43 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 175 : optimizer step time 0.40 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 175-------------------- +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6484 iter count: 1.0 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6484 iter count: 1.0 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6484 iter count: 1.0 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6484iter count: 1.0 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:17:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416504 41.497452 41.380020 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.264648 67.460938 67.199219 +util 91.000000 92.000000 90.000000 +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 176 : avg iteration time 47.23 seconds +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 176 : data loading time 0.00 +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 176 : avg forward pass time 12.69 +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 176 : backward pass time 10.14 +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 176 : optimizer step time 1.56 +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 176-------------------- +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:17:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416556 41.497669 41.380024 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.235352 67.343750 67.199219 +util 98.250000 99.000000 97.000000 +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 177 : avg iteration time 46.08 seconds +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 177 : data loading time 0.00 +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 177 : avg forward pass time 12.09 +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 177 : backward pass time 10.20 +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 177 : optimizer step time 1.57 +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 177-------------------- +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416883 41.498936 41.380024 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.157227 67.207031 67.140625 +util 91.250000 92.000000 90.000000 +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 178 : avg iteration time 43.52 seconds +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 178 : data loading time 0.00 +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 178 : avg forward pass time 10.82 +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 178 : backward pass time 10.17 +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 178 : optimizer step time 1.58 +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 178-------------------- +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:19:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416928 41.499130 41.380032 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.127930 67.148438 67.121094 +util 90.750000 92.000000 89.000000 +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 179 : avg iteration time 48.80 seconds +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 179 : data loading time 0.00 +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 179 : avg forward pass time 13.46 +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 179 : backward pass time 10.14 +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 179 : optimizer step time 1.58 +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 179-------------------- +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:20:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416866 41.498878 41.380032 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.521973 107.628906 107.486328 +util 61.750000 62.000000 61.000000 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 180 : avg iteration time 164.22 seconds +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 180 : data loading time 0.00 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 180 : avg forward pass time 73.48 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 180 : backward pass time 8.45 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 180 : optimizer step time 0.39 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 180-------------------- +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6982 iter count: 1.0 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6982 iter count: 1.0 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6982 iter count: 1.0 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6982iter count: 1.0 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:23:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416804 41.498577 41.380047 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.393066 67.441406 67.376953 +util 91.000000 92.000000 90.000000 +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 181 : avg iteration time 47.38 seconds +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 181 : data loading time 0.00 +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 181 : avg forward pass time 12.76 +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 181 : backward pass time 10.15 +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 181 : optimizer step time 1.57 +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 181-------------------- +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:23:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416802 41.498577 41.380039 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.552734 67.148438 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 182 : avg iteration time 46.12 seconds +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 182 : data loading time 0.00 +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 182 : avg forward pass time 12.11 +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 182 : backward pass time 10.18 +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 182 : optimizer step time 1.57 +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 182-------------------- +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:24:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416868 41.498814 41.380028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373535 67.396484 67.304688 +util 91.000000 100.000000 87.000000 +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 183 : avg iteration time 48.84 seconds +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 183 : data loading time 0.00 +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 183 : avg forward pass time 13.48 +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 183 : backward pass time 10.14 +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 183 : optimizer step time 1.58 +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 183-------------------- +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:25:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416860 41.498814 41.380028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.441406 67.396484 +util 93.750000 95.000000 92.000000 +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 184 : avg iteration time 43.47 seconds +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 184 : data loading time 0.00 +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 184 : avg forward pass time 10.82 +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 184 : backward pass time 10.13 +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 184 : optimizer step time 1.57 +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 184-------------------- +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:26:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416953 41.499187 41.380028 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.726562 107.564453 +util 70.250000 71.000000 69.000000 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 185 : avg iteration time 158.70 seconds +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 185 : data loading time 0.00 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 185 : avg forward pass time 70.69 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 185 : backward pass time 8.43 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 185 : optimizer step time 0.40 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 185-------------------- +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6387 iter count: 1.0 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6387 iter count: 1.0 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6387 iter count: 1.0 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6387iter count: 1.0 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:28:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416958 41.499191 41.380028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.508789 67.550781 67.382812 +util 96.750000 98.000000 95.000000 +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 186 : avg iteration time 50.14 seconds +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 186 : data loading time 0.00 +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 186 : avg forward pass time 14.13 +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 186 : backward pass time 10.16 +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 186 : optimizer step time 1.59 +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 186-------------------- +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:29:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416966 41.499187 41.380028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.049805 67.363281 66.945312 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 187 : avg iteration time 43.65 seconds +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 187 : data loading time 0.00 +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 187 : avg forward pass time 10.86 +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 187 : backward pass time 10.14 +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 187 : optimizer step time 1.57 +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 187-------------------- +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:30:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417025 41.499420 41.380028 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.240234 67.304688 67.218750 +util 89.500000 91.000000 88.000000 +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 188 : avg iteration time 43.87 seconds +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 188 : data loading time 0.00 +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 188 : avg forward pass time 11.00 +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 188 : backward pass time 10.15 +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 188 : optimizer step time 1.57 +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 188-------------------- +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:31:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417079 41.499649 41.380032 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.357422 67.433594 67.128906 +util 91.250000 92.000000 90.000000 +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 189 : avg iteration time 48.62 seconds +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 189 : data loading time 0.00 +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 189 : avg forward pass time 13.38 +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 189 : backward pass time 10.14 +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 189 : optimizer step time 1.58 +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 189-------------------- +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:31:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416861 41.498749 41.380032 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.692871 108.253906 107.505859 +util 62.750000 64.000000 61.000000 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 190 : avg iteration time 164.02 seconds +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 190 : data loading time 0.00 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 190 : avg forward pass time 73.40 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 190 : backward pass time 8.41 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 190 : optimizer step time 0.39 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 190-------------------- +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6504 iter count: 1.0 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6504 iter count: 1.0 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6504 iter count: 1.0 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6504iter count: 1.0 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:34:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416855 41.498730 41.380051 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.373535 67.376953 67.363281 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 191 : avg iteration time 50.08 seconds +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 191 : data loading time 0.00 +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 191 : avg forward pass time 14.11 +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 191 : backward pass time 10.15 +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 191 : optimizer step time 1.57 +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 191-------------------- +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0159 iter count: 1.0 +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:35:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416853 41.498730 41.380032 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.466309 67.494141 67.382812 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 192 : avg iteration time 43.41 seconds +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 192 : data loading time 0.00 +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 192 : avg forward pass time 10.79 +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 192 : backward pass time 10.12 +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 192 : optimizer step time 1.58 +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 192-------------------- +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:36:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416862 41.498756 41.380032 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.513672 67.382812 +util 97.500000 99.000000 96.000000 +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 193 : avg iteration time 48.96 seconds +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 193 : data loading time 0.00 +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 193 : avg forward pass time 13.51 +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 193 : backward pass time 10.13 +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 193 : optimizer step time 1.58 +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 193-------------------- +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:36:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416987 41.499245 41.380054 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.363770 67.382812 67.357422 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 194 : avg iteration time 48.64 seconds +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 194 : data loading time 0.00 +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 194 : avg forward pass time 13.39 +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 194 : backward pass time 10.13 +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 194 : optimizer step time 1.57 +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 194-------------------- +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:37:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416902 41.498890 41.380039 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.531738 107.667969 107.486328 +util 64.250000 65.000000 63.000000 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 195 : avg iteration time 155.93 seconds +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 195 : data loading time 0.00 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 195 : avg forward pass time 69.29 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 195 : backward pass time 8.50 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 195 : optimizer step time 0.39 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 195-------------------- +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6113 iter count: 1.0 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6113 iter count: 1.0 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6113 iter count: 1.0 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6113iter count: 1.0 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:40:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416910 41.498890 41.380062 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.393066 67.441406 67.376953 +util 92.250000 93.000000 91.000000 +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 196 : avg iteration time 49.78 seconds +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 196 : data loading time 0.00 +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 196 : avg forward pass time 13.95 +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 196 : backward pass time 10.15 +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 196 : optimizer step time 1.58 +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 196-------------------- +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:41:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416907 41.498901 41.380039 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.441406 67.396484 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 197 : avg iteration time 48.87 seconds +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 197 : data loading time 0.00 +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 197 : avg forward pass time 13.47 +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 197 : backward pass time 10.20 +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 197 : optimizer step time 1.57 +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 197-------------------- +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416899 41.498890 41.380039 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.480469 67.376953 +util 91.000000 100.000000 87.000000 +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 198 : avg iteration time 46.27 seconds +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 198 : data loading time 0.00 +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 198 : avg forward pass time 12.20 +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 198 : backward pass time 10.14 +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 198 : optimizer step time 1.57 +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 198-------------------- +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:42:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417157 41.499855 41.380062 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.490723 67.533203 67.363281 +util 95.000000 96.000000 93.000000 +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 199 : avg iteration time 46.09 seconds +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 199 : data loading time 0.00 +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 199 : avg forward pass time 12.12 +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 199 : backward pass time 10.15 +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 199 : optimizer step time 1.57 +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 199-------------------- +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:43:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.416933 41.498978 41.380062 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.667969 107.544922 +util 60.000000 64.000000 58.000000 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 200 : avg iteration time 163.56 seconds +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 200 : data loading time 0.00 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 200 : avg forward pass time 73.18 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 200 : backward pass time 8.40 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 200 : optimizer step time 0.41 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 200-------------------- +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6416 iter count: 1.0 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6416 iter count: 1.0 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6416 iter count: 1.0 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6416iter count: 1.0 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 17:46:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 17:46:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.34 seconds +[May 4, 2026 - 17:46:33 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 200 +[May 4, 2026 - 17:46:33 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000200 +[May 4, 2026 - 17:46:47 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 200 +[May 4, 2026 - 17:46:48 | INFO | fastgen.trainer:validate:358 ] Validation iteration 200 +[May 4, 2026 - 17:46:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:46:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 4, 2026 - 17:46:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.820 +[May 4, 2026 - 17:46:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:46:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:46:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.513 +[May 4, 2026 - 17:46:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:47:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:47:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.708 +[May 4, 2026 - 17:47:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:47:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.436 +[May 4, 2026 - 17:47:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:47:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:47:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.957 +[May 4, 2026 - 17:47:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:47:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:47:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.534 +[May 4, 2026 - 17:47:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:47:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:47:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.984 +[May 4, 2026 - 17:47:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:47:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:47:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.471 +[May 4, 2026 - 17:47:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:47:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:47:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.107 +[May 4, 2026 - 17:47:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 17:47:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 17:47:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.999 +[May 4, 2026 - 17:47:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 200-------------------- +[May 4, 2026 - 17:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 17:47:54 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 200 +[May 4, 2026 - 17:47:54 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.853 (n=10) +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419204 41.506611 41.380531 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.904297 68.730469 67.628906 +util 94.750000 96.000000 93.000000 +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 201 : avg iteration time 141.45 seconds +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 201 : data loading time 0.00 +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 201 : avg forward pass time 11.73 +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 201 : backward pass time 10.15 +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 201 : optimizer step time 1.56 +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 201-------------------- +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:48:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419219 41.506611 41.380554 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.104492 68.710938 67.902344 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 202 : avg iteration time 49.08 seconds +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 202 : data loading time 0.00 +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 202 : avg forward pass time 13.54 +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 202 : backward pass time 10.21 +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 202 : optimizer step time 1.56 +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 202-------------------- +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:49:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419029 41.505886 41.380554 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.084961 68.691406 67.882812 +util 98.250000 99.000000 97.000000 +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 203 : avg iteration time 46.16 seconds +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 203 : data loading time 0.00 +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 203 : avg forward pass time 12.12 +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 203 : backward pass time 10.19 +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 203 : optimizer step time 1.57 +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 203-------------------- +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:50:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419063 41.505890 41.380531 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.055664 68.691406 67.843750 +util 91.750000 93.000000 90.000000 +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 204 : avg iteration time 43.53 seconds +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 204 : data loading time 0.00 +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 204 : avg forward pass time 10.82 +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 204 : backward pass time 10.16 +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 204 : optimizer step time 1.58 +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 204-------------------- +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:50:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417664 41.500298 41.380539 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.088379 108.228516 107.667969 +util 62.250000 73.000000 58.000000 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 205 : avg iteration time 161.38 seconds +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 205 : data loading time 0.00 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 205 : avg forward pass time 72.06 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 205 : backward pass time 8.45 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 205 : optimizer step time 0.39 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 205-------------------- +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6250 iter count: 1.0 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6250 iter count: 1.0 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6250 iter count: 1.0 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6250iter count: 1.0 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:53:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417668 41.500298 41.380558 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.469727 67.871094 67.335938 +util 94.750000 96.000000 93.000000 +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 206 : avg iteration time 47.67 seconds +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 206 : data loading time 0.00 +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 206 : avg forward pass time 12.91 +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 206 : backward pass time 10.14 +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 206 : optimizer step time 1.56 +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 206-------------------- +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:54:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417724 41.500534 41.380543 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.871094 67.277344 +util 91.000000 92.000000 89.000000 +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 207 : avg iteration time 46.36 seconds +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 207 : data loading time 0.00 +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 207 : avg forward pass time 12.25 +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 207 : backward pass time 10.15 +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 207 : optimizer step time 1.56 +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 207-------------------- +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:55:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417909 41.501251 41.380543 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.445312 67.949219 67.277344 +util 98.750000 100.000000 97.000000 +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 208 : avg iteration time 46.10 seconds +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 208 : data loading time 0.00 +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 208 : avg forward pass time 12.10 +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 208 : backward pass time 10.16 +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 208 : optimizer step time 1.58 +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 208-------------------- +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:56:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417961 41.501480 41.380547 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.289062 67.675781 67.160156 +util 93.750000 100.000000 87.000000 +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 209 : avg iteration time 48.85 seconds +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 209 : data loading time 0.00 +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 209 : avg forward pass time 13.49 +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 209 : backward pass time 10.15 +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 209 : optimizer step time 1.57 +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 209-------------------- +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:56:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417488 41.499546 41.380569 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.726562 107.603516 +util 65.500000 67.000000 64.000000 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 210 : avg iteration time 158.33 seconds +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 210 : data loading time 0.00 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 210 : avg forward pass time 70.52 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 210 : backward pass time 8.45 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 210 : optimizer step time 0.40 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 210-------------------- +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6328 iter count: 1.0 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6328 iter count: 1.0 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6328 iter count: 1.0 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6328iter count: 1.0 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 17:59:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417480 41.499546 41.380558 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.464844 67.851562 67.335938 +util 98.750000 99.000000 98.000000 +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 211 : avg iteration time 47.23 seconds +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 211 : data loading time 0.00 +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 211 : avg forward pass time 12.70 +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 211 : backward pass time 10.14 +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 211 : optimizer step time 1.56 +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 211-------------------- +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:00:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417608 41.500023 41.380558 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.455078 67.753906 67.355469 +util 97.750000 99.000000 96.000000 +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 212 : avg iteration time 48.57 seconds +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 212 : data loading time 0.00 +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 212 : avg forward pass time 13.35 +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 212 : backward pass time 10.17 +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 212 : optimizer step time 1.56 +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 212-------------------- +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:01:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417839 41.500969 41.380558 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.464844 67.792969 67.355469 +util 89.750000 91.000000 88.000000 +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 213 : avg iteration time 43.51 seconds +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 213 : data loading time 0.00 +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 213 : avg forward pass time 10.83 +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 213 : backward pass time 10.15 +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 213 : optimizer step time 1.57 +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 213-------------------- +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:01:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417843 41.500969 41.380558 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.469727 67.812500 67.355469 +util 94.750000 96.000000 93.000000 +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 214 : avg iteration time 48.86 seconds +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 214 : data loading time 0.00 +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 214 : avg forward pass time 13.49 +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 214 : backward pass time 10.14 +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 214 : optimizer step time 1.56 +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 214-------------------- +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:02:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417550 41.499794 41.380558 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.561035 107.667969 107.525391 +util 65.500000 66.000000 64.000000 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 215 : avg iteration time 161.62 seconds +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 215 : data loading time 0.00 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 215 : avg forward pass time 72.19 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 215 : backward pass time 8.41 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 215 : optimizer step time 0.39 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 215-------------------- +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6602 iter count: 1.0 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6602 iter count: 1.0 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6602 iter count: 1.0 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6602iter count: 1.0 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:05:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417551 41.499794 41.380573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.500488 67.812500 67.396484 +util 98.000000 99.000000 96.000000 +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 216 : avg iteration time 47.63 seconds +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 216 : data loading time 0.00 +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 216 : avg forward pass time 12.87 +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 216 : backward pass time 10.15 +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 216 : optimizer step time 1.56 +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 216-------------------- +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:06:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417607 41.500027 41.380573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.529785 67.988281 67.376953 +util 99.500000 100.000000 99.000000 +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 217 : avg iteration time 46.04 seconds +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 217 : data loading time 0.00 +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 217 : avg forward pass time 12.08 +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 217 : backward pass time 10.13 +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 217 : optimizer step time 1.58 +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 217-------------------- +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:06:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417858 41.501030 41.380573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.773438 67.376953 +util 90.750000 92.000000 89.000000 +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 218 : avg iteration time 48.67 seconds +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 218 : data loading time 0.00 +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 218 : avg forward pass time 13.39 +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 218 : backward pass time 10.14 +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 218 : optimizer step time 1.58 +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 218-------------------- +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:07:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417867 41.501019 41.380573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.534668 68.007812 67.376953 +util 92.750000 94.000000 91.000000 +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 219 : avg iteration time 46.03 seconds +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 219 : data loading time 0.00 +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 219 : avg forward pass time 12.10 +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 219 : backward pass time 10.12 +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 219 : optimizer step time 1.57 +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 219-------------------- +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:08:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417590 41.499863 41.380604 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.570312 107.564453 +util 61.000000 63.000000 59.000000 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 220 : avg iteration time 160.80 seconds +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 220 : data loading time 0.00 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 220 : avg forward pass time 71.79 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 220 : backward pass time 8.42 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 220 : optimizer step time 0.39 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 220-------------------- +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6328 iter count: 1.0 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6328 iter count: 1.0 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6328 iter count: 1.0 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6328iter count: 1.0 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:11:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417599 41.499905 41.380604 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.533203 68.007812 67.375000 +util 95.000000 96.000000 93.000000 +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 221 : avg iteration time 44.63 seconds +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 221 : data loading time 0.00 +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 221 : avg forward pass time 11.36 +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 221 : backward pass time 10.16 +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 221 : optimizer step time 1.57 +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 221-------------------- +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:11:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417726 41.500423 41.380604 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.533203 68.125000 67.335938 +util 93.500000 100.000000 87.000000 +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 222 : avg iteration time 48.90 seconds +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 222 : data loading time 0.00 +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 222 : avg forward pass time 13.50 +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 222 : backward pass time 10.13 +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 222 : optimizer step time 1.57 +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 222-------------------- +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:12:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417910 41.501141 41.380608 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.518555 68.125000 67.316406 +util 91.250000 100.000000 88.000000 +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 223 : avg iteration time 48.61 seconds +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 223 : data loading time 0.00 +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 223 : avg forward pass time 13.39 +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 223 : backward pass time 10.13 +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 223 : optimizer step time 1.55 +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 223-------------------- +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:13:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417910 41.501141 41.380608 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.513672 67.988281 67.355469 +util 94.750000 96.000000 93.000000 +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 224 : avg iteration time 43.49 seconds +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 224 : data loading time 0.00 +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 224 : avg forward pass time 10.81 +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 224 : backward pass time 10.16 +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 224 : optimizer step time 1.57 +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 224-------------------- +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:14:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417494 41.499477 41.380608 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.628906 107.623047 +util 62.250000 64.000000 60.000000 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 225 : avg iteration time 166.18 seconds +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 225 : data loading time 0.00 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 225 : avg forward pass time 74.49 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 225 : backward pass time 8.45 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 225 : optimizer step time 0.39 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 225-------------------- +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6133 iter count: 1.0 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6133 iter count: 1.0 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6133 iter count: 1.0 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6133iter count: 1.0 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:16:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417497 41.499454 41.380630 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.430664 67.773438 67.316406 +util 90.500000 92.000000 88.000000 +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 226 : avg iteration time 49.76 seconds +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 226 : data loading time 0.00 +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 226 : avg forward pass time 13.97 +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 226 : backward pass time 10.13 +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 226 : optimizer step time 1.57 +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 226-------------------- +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:17:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417684 41.500210 41.380611 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.367188 67.460938 67.335938 +util 96.000000 98.000000 94.000000 +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 227 : avg iteration time 48.67 seconds +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 227 : data loading time 0.00 +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 227 : avg forward pass time 13.42 +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 227 : backward pass time 10.12 +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 227 : optimizer step time 1.56 +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 227-------------------- +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:18:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418208 41.502182 41.380650 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.425781 67.695312 67.335938 +util 95.750000 97.000000 94.000000 +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 228 : avg iteration time 43.40 seconds +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 228 : data loading time 0.00 +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 228 : avg forward pass time 10.79 +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 228 : backward pass time 10.13 +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 228 : optimizer step time 1.56 +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 228-------------------- +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:19:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418320 41.502613 41.380642 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.386719 67.480469 67.355469 +util 96.500000 98.000000 94.000000 +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 229 : avg iteration time 48.61 seconds +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 229 : data loading time 0.00 +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 229 : avg forward pass time 13.39 +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 229 : backward pass time 10.11 +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 229 : optimizer step time 1.58 +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 229-------------------- +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:20:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417678 41.500042 41.380630 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.565918 107.628906 107.544922 +util 70.500000 72.000000 69.000000 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 230 : avg iteration time 157.88 seconds +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 230 : data loading time 0.00 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 230 : avg forward pass time 70.35 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 230 : backward pass time 8.39 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 230 : optimizer step time 0.40 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 230-------------------- +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6445 iter count: 1.0 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6445 iter count: 1.0 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6445 iter count: 1.0 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6445iter count: 1.0 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:22:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417705 41.500023 41.380722 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.505371 67.832031 67.396484 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 231 : avg iteration time 49.96 seconds +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 231 : data loading time 0.00 +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 231 : avg forward pass time 14.07 +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 231 : backward pass time 10.14 +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 231 : optimizer step time 1.56 +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 231-------------------- +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:23:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417824 41.500500 41.380722 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.851562 67.318359 +util 92.000000 100.000000 89.000000 +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 232 : avg iteration time 46.28 seconds +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 232 : data loading time 0.00 +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 232 : avg forward pass time 12.20 +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 232 : backward pass time 10.15 +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 232 : optimizer step time 1.57 +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 232-------------------- +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:24:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418220 41.502041 41.380722 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.851562 67.318359 +util 99.000000 100.000000 98.000000 +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 233 : avg iteration time 43.53 seconds +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 233 : data loading time 0.00 +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 233 : avg forward pass time 10.80 +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 233 : backward pass time 10.17 +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 233 : optimizer step time 1.58 +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 233-------------------- +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:25:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418269 41.502243 41.380745 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.451660 67.851562 67.318359 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 234 : avg iteration time 46.02 seconds +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 234 : data loading time 0.00 +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 234 : avg forward pass time 12.08 +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 234 : backward pass time 10.13 +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 234 : optimizer step time 1.58 +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 234-------------------- +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:25:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417888 41.500736 41.380726 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.604980 107.726562 107.564453 +util 65.500000 67.000000 64.000000 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 235 : avg iteration time 158.34 seconds +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 235 : data loading time 0.00 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 235 : avg forward pass time 70.54 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 235 : backward pass time 8.43 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 235 : optimizer step time 0.39 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 235-------------------- +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6230 iter count: 1.0 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6230 iter count: 1.0 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6230 iter count: 1.0 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6230iter count: 1.0 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:28:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417773 41.500278 41.380726 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.450195 67.734375 67.355469 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 236 : avg iteration time 44.66 seconds +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 236 : data loading time 0.00 +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 236 : avg forward pass time 11.41 +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 236 : backward pass time 10.13 +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 236 : optimizer step time 1.57 +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 236-------------------- +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:29:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417775 41.500278 41.380733 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.812500 67.375000 +util 98.500000 100.000000 96.000000 +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 237 : avg iteration time 43.56 seconds +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 237 : data loading time 0.00 +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 237 : avg forward pass time 10.84 +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 237 : backward pass time 10.15 +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 237 : optimizer step time 1.57 +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 237-------------------- +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:29:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417771 41.500256 41.380733 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.489258 67.832031 67.375000 +util 93.250000 99.000000 87.000000 +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 238 : avg iteration time 48.66 seconds +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 238 : data loading time 0.00 +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 238 : avg forward pass time 13.40 +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 238 : backward pass time 10.16 +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 238 : optimizer step time 1.57 +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 238-------------------- +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:30:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417771 41.500256 41.380733 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.812500 67.375000 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 239 : avg iteration time 48.79 seconds +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 239 : data loading time 0.00 +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 239 : avg forward pass time 13.46 +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 239 : backward pass time 10.15 +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 239 : optimizer step time 1.58 +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 239-------------------- +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:31:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417844 41.500523 41.380756 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.736816 107.740234 107.726562 +util 68.750000 73.000000 62.000000 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 240 : avg iteration time 160.61 seconds +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 240 : data loading time 0.00 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 240 : avg forward pass time 71.71 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 240 : backward pass time 8.45 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 240 : optimizer step time 0.39 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 240-------------------- +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6592 iter count: 1.0 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6592 iter count: 1.0 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6592 iter count: 1.0 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6592iter count: 1.0 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:34:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417755 41.500191 41.380733 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.484375 67.871094 67.355469 +util 91.250000 92.000000 89.000000 +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 241 : avg iteration time 47.26 seconds +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 241 : data loading time 0.00 +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 241 : avg forward pass time 12.68 +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 241 : backward pass time 10.14 +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 241 : optimizer step time 1.56 +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 241-------------------- +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:35:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417755 41.500191 41.380733 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.469727 67.871094 67.335938 +util 99.000000 100.000000 98.000000 +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 242 : avg iteration time 43.52 seconds +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 242 : data loading time 0.00 +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 242 : avg forward pass time 10.76 +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 242 : backward pass time 10.14 +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 242 : optimizer step time 1.58 +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 242-------------------- +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:35:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417756 41.500191 41.380737 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.459961 67.832031 67.335938 +util 98.250000 99.000000 96.000000 +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 243 : avg iteration time 46.17 seconds +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 243 : data loading time 0.00 +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 243 : avg forward pass time 12.15 +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 243 : backward pass time 10.14 +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 243 : optimizer step time 1.59 +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 243-------------------- +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:36:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417756 41.500191 41.380737 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.474609 67.890625 67.335938 +util 97.500000 99.000000 95.000000 +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 244 : avg iteration time 43.39 seconds +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 244 : data loading time 0.00 +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 244 : avg forward pass time 10.77 +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 244 : backward pass time 10.15 +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 244 : optimizer step time 1.56 +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 244-------------------- +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:37:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417597 41.499546 41.380737 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.707031 107.505859 +util 68.500000 70.000000 66.000000 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 245 : avg iteration time 158.03 seconds +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 245 : data loading time 0.00 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 245 : avg forward pass time 70.41 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 245 : backward pass time 8.41 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 245 : optimizer step time 0.39 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 245-------------------- +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6484 iter count: 1.0 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6484 iter count: 1.0 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6484 iter count: 1.0 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6484iter count: 1.0 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:39:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417597 41.499546 41.380737 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.528320 67.988281 67.375000 +util 97.000000 100.000000 88.000000 +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 246 : avg iteration time 47.24 seconds +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 246 : data loading time 0.00 +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 246 : avg forward pass time 12.69 +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 246 : backward pass time 10.14 +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 246 : optimizer step time 1.56 +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 246-------------------- +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:40:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417605 41.499546 41.380753 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 68.027344 67.316406 +util 93.500000 100.000000 87.000000 +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 247 : avg iteration time 46.01 seconds +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 247 : data loading time 0.00 +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 247 : avg forward pass time 12.06 +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 247 : backward pass time 10.15 +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 247 : optimizer step time 1.58 +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 247-------------------- +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:41:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417680 41.499790 41.380783 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.479492 67.968750 67.316406 +util 98.750000 100.000000 97.000000 +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 248 : avg iteration time 48.75 seconds +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 248 : data loading time 0.00 +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 248 : avg forward pass time 13.43 +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 248 : backward pass time 10.14 +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 248 : optimizer step time 1.58 +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 248-------------------- +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:42:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417976 41.501015 41.380749 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.474609 67.949219 67.316406 +util 96.500000 98.000000 94.000000 +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 249 : avg iteration time 45.97 seconds +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 249 : data loading time 0.00 +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 249 : avg forward pass time 12.06 +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 249 : backward pass time 10.14 +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 249 : optimizer step time 1.58 +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 249-------------------- +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:43:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417725 41.499973 41.380795 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.756348 108.214844 107.603516 +util 67.000000 68.000000 65.000000 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 250 : avg iteration time 166.22 seconds +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 250 : data loading time 0.00 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 250 : avg forward pass time 74.52 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 250 : backward pass time 8.40 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 250 : optimizer step time 0.39 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 250-------------------- +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6084 iter count: 1.0 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6084 iter count: 1.0 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6084 iter count: 1.0 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6084iter count: 1.0 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:45:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417714 41.499973 41.380768 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.464844 67.910156 67.316406 +util 97.500000 99.000000 95.000000 +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 251 : avg iteration time 49.80 seconds +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 251 : data loading time 0.00 +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 251 : avg forward pass time 13.95 +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 251 : backward pass time 10.16 +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 251 : optimizer step time 1.59 +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 251-------------------- +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:46:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417721 41.499977 41.380768 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.337891 67.753906 67.199219 +util 90.500000 92.000000 88.000000 +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 252 : avg iteration time 48.72 seconds +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 252 : data loading time 0.00 +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 252 : avg forward pass time 13.42 +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 252 : backward pass time 10.14 +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 252 : optimizer step time 1.58 +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 252-------------------- +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:47:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417727 41.499977 41.380791 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.871094 67.296875 +util 92.000000 93.000000 90.000000 +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 253 : avg iteration time 46.02 seconds +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 253 : data loading time 0.00 +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 253 : avg forward pass time 12.09 +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 253 : backward pass time 10.13 +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 253 : optimizer step time 1.56 +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 253-------------------- +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:48:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417911 41.500710 41.380795 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.871094 67.296875 +util 97.500000 99.000000 95.000000 +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 254 : avg iteration time 46.26 seconds +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 254 : data loading time 0.00 +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 254 : avg forward pass time 12.21 +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 254 : backward pass time 10.12 +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 254 : optimizer step time 1.56 +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 254-------------------- +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:49:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417720 41.499836 41.380772 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.726562 107.525391 +util 66.500000 68.000000 65.000000 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 255 : avg iteration time 160.82 seconds +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 255 : data loading time 0.00 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 255 : avg forward pass time 71.78 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 255 : backward pass time 8.40 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 255 : optimizer step time 0.40 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 255-------------------- +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6562 iter count: 1.0 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6562 iter count: 1.0 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6562 iter count: 1.0 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6562iter count: 1.0 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:51:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417711 41.499847 41.380772 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.518555 67.832031 67.414062 +util 97.500000 99.000000 95.000000 +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 256 : avg iteration time 47.19 seconds +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 256 : data loading time 0.00 +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 256 : avg forward pass time 12.65 +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 256 : backward pass time 10.15 +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 256 : optimizer step time 1.58 +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 256-------------------- +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:52:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417727 41.499855 41.380802 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.303711 67.675781 67.179688 +util 92.750000 99.000000 86.000000 +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 257 : avg iteration time 43.46 seconds +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 257 : data loading time 0.00 +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 257 : avg forward pass time 10.79 +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 257 : backward pass time 10.14 +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 257 : optimizer step time 1.57 +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 257-------------------- +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:53:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417719 41.499851 41.380798 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.318359 67.675781 67.199219 +util 94.500000 96.000000 92.000000 +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 258 : avg iteration time 48.70 seconds +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 258 : data loading time 0.00 +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 258 : avg forward pass time 13.42 +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 258 : backward pass time 10.13 +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 258 : optimizer step time 1.58 +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 258-------------------- +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:54:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417742 41.499851 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.303711 67.675781 67.179688 +util 97.750000 99.000000 95.000000 +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 259 : avg iteration time 46.23 seconds +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 259 : data loading time 0.00 +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 259 : avg forward pass time 12.18 +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 259 : backward pass time 10.14 +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 259 : optimizer step time 1.58 +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 259-------------------- +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:54:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417669 41.499565 41.380787 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.521973 107.628906 107.486328 +util 62.750000 64.000000 61.000000 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 260 : avg iteration time 158.34 seconds +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 260 : data loading time 0.00 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 260 : avg forward pass time 70.56 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 260 : backward pass time 8.40 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 260 : optimizer step time 0.39 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 260-------------------- +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:57:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417672 41.499565 41.380798 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.440430 67.695312 67.355469 +util 96.000000 98.000000 93.000000 +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 261 : avg iteration time 44.63 seconds +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 261 : data loading time 0.00 +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 261 : avg forward pass time 11.39 +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 261 : backward pass time 10.14 +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 261 : optimizer step time 1.56 +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 261-------------------- +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:58:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417666 41.499565 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.450195 67.792969 67.335938 +util 93.500000 95.000000 91.000000 +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 262 : avg iteration time 48.57 seconds +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 262 : data loading time 0.00 +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 262 : avg forward pass time 13.35 +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 262 : backward pass time 10.14 +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 262 : optimizer step time 1.58 +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 262-------------------- +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:58:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417670 41.499565 41.380791 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.450195 67.792969 67.335938 +util 98.750000 100.000000 97.000000 +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 263 : avg iteration time 43.46 seconds +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 263 : data loading time 0.00 +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 263 : avg forward pass time 10.80 +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 263 : backward pass time 10.14 +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 263 : optimizer step time 1.58 +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 263-------------------- +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 18:59:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417668 41.499565 41.380783 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.450195 67.792969 67.335938 +util 97.500000 99.000000 95.000000 +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 264 : avg iteration time 45.98 seconds +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 264 : data loading time 0.00 +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 264 : avg forward pass time 12.08 +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 264 : backward pass time 10.11 +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 264 : optimizer step time 1.57 +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 264-------------------- +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:00:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417528 41.498966 41.380798 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.687500 107.642578 +util 65.250000 67.000000 63.000000 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 265 : avg iteration time 157.98 seconds +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 265 : data loading time 0.00 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 265 : avg forward pass time 70.36 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 265 : backward pass time 8.44 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 265 : optimizer step time 0.39 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 265-------------------- +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5923 iter count: 1.0 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5923 iter count: 1.0 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5923 iter count: 1.0 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5923iter count: 1.0 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:03:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417520 41.498966 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.601562 67.812500 67.531250 +util 98.250000 99.000000 96.000000 +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 266 : avg iteration time 47.23 seconds +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 266 : data loading time 0.00 +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 266 : avg forward pass time 12.69 +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 266 : backward pass time 10.13 +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 266 : optimizer step time 1.57 +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 266-------------------- +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:03:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417643 41.499458 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.689453 67.753906 67.667969 +util 99.000000 99.000000 99.000000 +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 267 : avg iteration time 46.64 seconds +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 267 : data loading time 0.00 +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 267 : avg forward pass time 12.28 +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 267 : backward pass time 10.14 +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 267 : optimizer step time 1.57 +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 267-------------------- +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:04:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417716 41.499702 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777344 67.785156 67.753906 +util 97.750000 99.000000 95.000000 +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 268 : avg iteration time 48.54 seconds +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 268 : data loading time 0.00 +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 268 : avg forward pass time 13.34 +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 268 : backward pass time 10.14 +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 268 : optimizer step time 1.57 +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 268-------------------- +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:05:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417722 41.499702 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.835938 67.843750 67.812500 +util 96.000000 98.000000 93.000000 +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 269 : avg iteration time 43.38 seconds +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 269 : data loading time 0.00 +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 269 : avg forward pass time 10.76 +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 269 : backward pass time 10.14 +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 269 : optimizer step time 1.58 +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 269-------------------- +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:06:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417499 41.498829 41.380775 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.804688 107.603516 +util 69.000000 70.000000 67.000000 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 270 : avg iteration time 158.04 seconds +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 270 : data loading time 0.00 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 270 : avg forward pass time 70.41 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 270 : backward pass time 8.39 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 270 : optimizer step time 0.39 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 270-------------------- +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6240 iter count: 1.0 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6240 iter count: 1.0 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6240 iter count: 1.0 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6240iter count: 1.0 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:08:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417503 41.498833 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.471191 67.812500 67.357422 +util 92.250000 100.000000 89.000000 +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 271 : avg iteration time 50.26 seconds +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 271 : data loading time 0.00 +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 271 : avg forward pass time 14.20 +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 271 : backward pass time 10.13 +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 271 : optimizer step time 1.58 +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 271-------------------- +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:09:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417572 41.499084 41.380787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.437012 67.792969 67.318359 +util 97.750000 99.000000 95.000000 +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 272 : avg iteration time 43.31 seconds +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 272 : data loading time 0.00 +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 272 : avg forward pass time 10.74 +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 272 : backward pass time 10.12 +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 272 : optimizer step time 1.57 +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 272-------------------- +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:10:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417877 41.500317 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.563965 68.183594 67.357422 +util 98.000000 99.000000 95.000000 +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 273 : avg iteration time 48.60 seconds +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 273 : data loading time 0.00 +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 273 : avg forward pass time 13.36 +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 273 : backward pass time 10.14 +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 273 : optimizer step time 1.58 +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 273-------------------- +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:11:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417877 41.500317 41.380775 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.446777 67.832031 67.318359 +util 92.250000 94.000000 89.000000 +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 274 : avg iteration time 43.55 seconds +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 274 : data loading time 0.00 +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 274 : avg forward pass time 10.83 +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 274 : backward pass time 10.15 +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 274 : optimizer step time 1.59 +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 274-------------------- +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:11:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417858 41.500214 41.380795 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.663574 107.726562 107.642578 +util 62.500000 64.000000 60.000000 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 275 : avg iteration time 161.19 seconds +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 275 : data loading time 0.00 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 275 : avg forward pass time 71.98 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 275 : backward pass time 8.44 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 275 : optimizer step time 0.39 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 275-------------------- +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6250 iter count: 1.0 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6250 iter count: 1.0 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6250 iter count: 1.0 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6250iter count: 1.0 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:14:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417728 41.499702 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.520020 67.949219 67.376953 +util 97.250000 99.000000 94.000000 +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 276 : avg iteration time 49.84 seconds +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 276 : data loading time 0.00 +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 276 : avg forward pass time 13.95 +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 276 : backward pass time 10.15 +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 276 : optimizer step time 1.57 +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 276-------------------- +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:15:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417730 41.499725 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.642090 68.437500 67.376953 +util 98.750000 100.000000 97.000000 +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 277 : avg iteration time 48.63 seconds +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 277 : data loading time 0.00 +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 277 : avg forward pass time 13.38 +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 277 : backward pass time 10.14 +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 277 : optimizer step time 1.58 +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 277-------------------- +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:16:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417789 41.499962 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.617676 68.281250 67.396484 +util 96.750000 100.000000 89.000000 +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 278 : avg iteration time 48.84 seconds +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 278 : data loading time 0.00 +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 278 : avg forward pass time 13.46 +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 278 : backward pass time 10.16 +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 278 : optimizer step time 1.57 +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 278-------------------- +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:17:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417789 41.499962 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.612793 68.320312 67.376953 +util 92.250000 99.000000 89.000000 +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 279 : avg iteration time 43.44 seconds +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 279 : data loading time 0.00 +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 279 : avg forward pass time 10.78 +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 279 : backward pass time 10.13 +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 279 : optimizer step time 1.57 +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 279-------------------- +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417792 41.499962 41.380779 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.658691 107.765625 107.623047 +util 62.000000 64.000000 59.000000 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 280 : avg iteration time 163.04 seconds +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 280 : data loading time 0.00 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 280 : avg forward pass time 72.91 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 280 : backward pass time 8.42 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 280 : optimizer step time 0.38 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 280-------------------- +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6143 iter count: 1.0 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6143 iter count: 1.0 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6143 iter count: 1.0 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6143iter count: 1.0 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:20:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417705 41.499607 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.437012 67.734375 67.337891 +util 99.000000 100.000000 97.000000 +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 281 : avg iteration time 44.48 seconds +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 281 : data loading time 0.00 +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 281 : avg forward pass time 11.31 +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 281 : backward pass time 10.13 +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 281 : optimizer step time 1.58 +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 281-------------------- +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:21:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417704 41.499607 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.890625 67.337891 +util 95.500000 97.000000 93.000000 +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 282 : avg iteration time 48.76 seconds +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 282 : data loading time 0.00 +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 282 : avg forward pass time 13.44 +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 282 : backward pass time 10.16 +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 282 : optimizer step time 1.58 +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 282-------------------- +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:22:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417767 41.499859 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.461426 67.890625 67.318359 +util 91.250000 99.000000 88.000000 +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 283 : avg iteration time 48.70 seconds +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 283 : data loading time 0.00 +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 283 : avg forward pass time 13.41 +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 283 : backward pass time 10.13 +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 283 : optimizer step time 1.56 +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 283-------------------- +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:22:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417835 41.500134 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.437012 67.792969 67.318359 +util 94.250000 96.000000 91.000000 +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 284 : avg iteration time 46.08 seconds +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 284 : data loading time 0.00 +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 284 : avg forward pass time 12.11 +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 284 : backward pass time 10.15 +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 284 : optimizer step time 1.57 +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 284-------------------- +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:23:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417836 41.500114 41.380798 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.731934 107.824219 107.701172 +util 62.750000 65.000000 60.000000 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 285 : avg iteration time 160.79 seconds +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 285 : data loading time 0.00 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 285 : avg forward pass time 71.82 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 285 : backward pass time 8.40 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 285 : optimizer step time 0.39 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 285-------------------- +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6484 iter count: 1.0 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6484 iter count: 1.0 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6484 iter count: 1.0 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6484iter count: 1.0 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:26:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417787 41.499935 41.380779 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.485840 67.871094 67.357422 +util 91.750000 100.000000 88.000000 +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 286 : avg iteration time 44.74 seconds +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 286 : data loading time 0.00 +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 286 : avg forward pass time 11.45 +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 286 : backward pass time 10.12 +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 286 : optimizer step time 1.57 +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 286-------------------- +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:27:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417783 41.499916 41.380783 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.549316 68.007812 67.396484 +util 90.250000 99.000000 87.000000 +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 287 : avg iteration time 48.59 seconds +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 287 : data loading time 0.00 +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 287 : avg forward pass time 13.37 +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 287 : backward pass time 10.15 +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 287 : optimizer step time 1.56 +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 287-------------------- +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:27:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417782 41.499912 41.380783 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.559082 68.046875 67.396484 +util 92.750000 100.000000 90.000000 +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 288 : avg iteration time 48.64 seconds +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 288 : data loading time 0.00 +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 288 : avg forward pass time 13.38 +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 288 : backward pass time 10.14 +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 288 : optimizer step time 1.57 +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 288-------------------- +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:28:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417782 41.499912 41.380783 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.554199 68.027344 67.396484 +util 97.250000 99.000000 94.000000 +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 289 : avg iteration time 48.70 seconds +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 289 : data loading time 0.00 +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 289 : avg forward pass time 13.41 +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 289 : backward pass time 10.14 +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 289 : optimizer step time 1.58 +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 289-------------------- +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:29:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417847 41.500164 41.380787 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.673340 107.681641 107.648438 +util 62.250000 72.000000 58.000000 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 290 : avg iteration time 163.52 seconds +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 290 : data loading time 0.00 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 290 : avg forward pass time 73.15 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 290 : backward pass time 8.46 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 290 : optimizer step time 0.40 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 290-------------------- +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6084 iter count: 1.0 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6084 iter count: 1.0 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6084 iter count: 1.0 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6084iter count: 1.0 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:32:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417799 41.499969 41.380787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.479492 67.910156 67.335938 +util 92.500000 99.000000 90.000000 +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 291 : avg iteration time 44.60 seconds +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 291 : data loading time 0.00 +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 291 : avg forward pass time 11.38 +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 291 : backward pass time 10.14 +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 291 : optimizer step time 1.58 +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 291-------------------- +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:32:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417804 41.499969 41.380806 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.499023 67.988281 67.335938 +util 99.000000 100.000000 98.000000 +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 292 : avg iteration time 48.75 seconds +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 292 : data loading time 0.00 +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 292 : avg forward pass time 13.42 +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 292 : backward pass time 10.17 +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 292 : optimizer step time 1.56 +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 292-------------------- +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:33:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417797 41.499947 41.380787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 68.027344 67.316406 +util 91.750000 93.000000 89.000000 +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 293 : avg iteration time 46.09 seconds +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 293 : data loading time 0.00 +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 293 : avg forward pass time 12.08 +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 293 : backward pass time 10.14 +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 293 : optimizer step time 1.57 +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 293-------------------- +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:34:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417925 41.500469 41.380787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.494141 67.910156 67.355469 +util 92.000000 94.000000 89.000000 +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 294 : avg iteration time 46.01 seconds +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 294 : data loading time 0.00 +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 294 : avg forward pass time 12.08 +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 294 : backward pass time 10.13 +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 294 : optimizer step time 1.57 +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 294-------------------- +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:35:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417842 41.500130 41.380787 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.687500 107.564453 +util 65.000000 67.000000 62.000000 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 295 : avg iteration time 163.66 seconds +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 295 : data loading time 0.00 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 295 : avg forward pass time 73.19 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 295 : backward pass time 8.42 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 295 : optimizer step time 0.40 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 295-------------------- +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5957 iter count: 1.0 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5957 iter count: 1.0 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5957 iter count: 1.0 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5957iter count: 1.0 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:38:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417838 41.500107 41.380787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.520020 67.890625 67.396484 +util 91.250000 99.000000 88.000000 +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 296 : avg iteration time 44.49 seconds +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 296 : data loading time 0.00 +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 296 : avg forward pass time 11.31 +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 296 : backward pass time 10.15 +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 296 : optimizer step time 1.57 +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 296-------------------- +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:38:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417837 41.500107 41.380787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.588379 67.871094 67.494141 +util 91.500000 99.000000 88.000000 +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 297 : avg iteration time 43.41 seconds +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 297 : data loading time 0.00 +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 297 : avg forward pass time 10.77 +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 297 : backward pass time 10.15 +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 297 : optimizer step time 1.58 +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 297-------------------- +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:39:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417849 41.500153 41.380787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.480957 67.910156 67.337891 +util 91.750000 100.000000 88.000000 +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 298 : avg iteration time 48.80 seconds +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 298 : data loading time 0.00 +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 298 : avg forward pass time 13.42 +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 298 : backward pass time 10.16 +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 298 : optimizer step time 1.56 +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 298-------------------- +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:40:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418032 41.500862 41.380787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.397949 67.871094 67.240234 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 299 : avg iteration time 46.14 seconds +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 299 : data loading time 0.00 +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 299 : avg forward pass time 12.15 +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 299 : backward pass time 10.15 +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 299 : optimizer step time 1.57 +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 299-------------------- +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:41:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417676 41.499439 41.380787 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.687500 107.583984 +util 68.500000 70.000000 66.000000 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 300 : avg iteration time 160.81 seconds +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 300 : data loading time 0.00 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 300 : avg forward pass time 71.81 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 300 : backward pass time 8.42 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 300 : optimizer step time 0.40 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 300-------------------- +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 19:43:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 19:44:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.33 seconds +[May 4, 2026 - 19:44:00 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 300 +[May 4, 2026 - 19:44:00 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000300 +[May 4, 2026 - 19:44:16 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 300 +[May 4, 2026 - 19:44:16 | INFO | fastgen.trainer:validate:358 ] Validation iteration 300 +[May 4, 2026 - 19:44:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 4, 2026 - 19:44:22 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.538 +[May 4, 2026 - 19:44:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 19:44:27 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.137 +[May 4, 2026 - 19:44:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 19:44:32 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.509 +[May 4, 2026 - 19:44:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 19:44:37 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.779 +[May 4, 2026 - 19:44:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 19:44:42 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.993 +[May 4, 2026 - 19:44:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 19:44:47 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.730 +[May 4, 2026 - 19:44:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 19:44:52 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.758 +[May 4, 2026 - 19:44:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 19:44:57 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.966 +[May 4, 2026 - 19:44:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:44:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 19:45:01 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.871 +[May 4, 2026 - 19:45:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 19:45:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 19:45:07 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.519 +[May 4, 2026 - 19:45:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 300-------------------- +[May 4, 2026 - 19:45:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 19:45:23 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 300 +[May 4, 2026 - 19:45:23 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.580 (n=10) +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419343 41.504742 41.381229 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.089844 68.214844 67.714844 +util 92.500000 94.000000 89.000000 +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 301 : avg iteration time 142.71 seconds +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 301 : data loading time 0.00 +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 301 : avg forward pass time 11.73 +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 301 : backward pass time 10.14 +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 301 : optimizer step time 1.57 +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 301-------------------- +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:46:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419349 41.504765 41.381229 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.084961 68.273438 67.519531 +util 93.750000 100.000000 91.000000 +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 302 : avg iteration time 46.19 seconds +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 302 : data loading time 0.00 +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 302 : avg forward pass time 12.12 +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 302 : backward pass time 10.18 +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 302 : optimizer step time 1.59 +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 302-------------------- +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:46:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419346 41.504742 41.381229 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.099609 68.273438 67.578125 +util 99.250000 100.000000 99.000000 +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 303 : avg iteration time 46.13 seconds +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 303 : data loading time 0.00 +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 303 : avg forward pass time 12.11 +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 303 : backward pass time 10.14 +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 303 : optimizer step time 1.57 +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 303-------------------- +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:47:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418839 41.502724 41.381233 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.163086 68.234375 67.949219 +util 92.250000 99.000000 89.000000 +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 304 : avg iteration time 48.84 seconds +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 304 : data loading time 0.00 +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 304 : avg forward pass time 13.46 +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 304 : backward pass time 10.19 +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 304 : optimizer step time 1.56 +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 304-------------------- +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:48:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418567 41.501633 41.381233 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.546387 107.609375 107.525391 +util 62.000000 71.000000 58.000000 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 305 : avg iteration time 166.41 seconds +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 305 : data loading time 0.00 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 305 : avg forward pass time 74.60 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 305 : backward pass time 8.39 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 305 : optimizer step time 0.39 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 305-------------------- +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6553 iter count: 1.0 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6553 iter count: 1.0 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6553 iter count: 1.0 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6553iter count: 1.0 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:51:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418572 41.501633 41.381233 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.601562 67.628906 67.519531 +util 96.750000 98.000000 94.000000 +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 306 : avg iteration time 44.49 seconds +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 306 : data loading time 0.00 +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 306 : avg forward pass time 11.30 +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 306 : backward pass time 10.14 +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 306 : optimizer step time 1.59 +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 306-------------------- +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:52:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418640 41.501877 41.381237 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.625977 67.667969 67.500000 +util 96.250000 98.000000 93.000000 +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 307 : avg iteration time 45.99 seconds +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 307 : data loading time 0.00 +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 307 : avg forward pass time 12.04 +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 307 : backward pass time 10.21 +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 307 : optimizer step time 1.56 +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 307-------------------- +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:52:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418631 41.501877 41.381237 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.728516 67.746094 67.675781 +util 98.750000 100.000000 97.000000 +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 308 : avg iteration time 46.05 seconds +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 308 : data loading time 0.00 +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 308 : avg forward pass time 12.10 +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 308 : backward pass time 10.13 +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 308 : optimizer step time 1.57 +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 308-------------------- +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:53:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418631 41.501877 41.381237 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.611328 67.648438 67.500000 +util 92.750000 95.000000 89.000000 +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 309 : avg iteration time 46.11 seconds +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 309 : data loading time 0.00 +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 309 : avg forward pass time 12.12 +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 309 : backward pass time 10.13 +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 309 : optimizer step time 1.58 +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 309-------------------- +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:54:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418530 41.501450 41.381241 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.644043 107.662109 107.589844 +util 61.250000 70.000000 55.000000 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 310 : avg iteration time 158.14 seconds +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 310 : data loading time 0.00 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 310 : avg forward pass time 70.47 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 310 : backward pass time 8.45 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 310 : optimizer step time 0.40 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 310-------------------- +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5938 iter count: 1.0 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5938 iter count: 1.0 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5938 iter count: 1.0 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5938iter count: 1.0 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:56:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418530 41.501450 41.381241 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.324707 67.343750 67.318359 +util 98.750000 100.000000 96.000000 +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 311 : avg iteration time 47.15 seconds +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 311 : data loading time 0.00 +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 311 : avg forward pass time 12.66 +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 311 : backward pass time 10.14 +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 311 : optimizer step time 1.55 +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 311-------------------- +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:57:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418657 41.501942 41.381248 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.354004 67.519531 67.298828 +util 98.750000 99.000000 98.000000 +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 312 : avg iteration time 46.02 seconds +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 312 : data loading time 0.00 +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 312 : avg forward pass time 12.07 +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 312 : backward pass time 10.17 +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 312 : optimizer step time 1.57 +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 312-------------------- +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:58:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418669 41.501942 41.381268 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.441895 67.460938 67.435547 +util 93.250000 100.000000 90.000000 +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 313 : avg iteration time 46.08 seconds +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 313 : data loading time 0.00 +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 313 : avg forward pass time 12.09 +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 313 : backward pass time 10.16 +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 313 : optimizer step time 1.59 +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 313-------------------- +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:59:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418663 41.501942 41.381245 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.388184 67.480469 67.357422 +util 92.750000 95.000000 89.000000 +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 314 : avg iteration time 43.42 seconds +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 314 : data loading time 0.00 +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 314 : avg forward pass time 10.77 +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 314 : backward pass time 10.13 +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 314 : optimizer step time 1.59 +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 314-------------------- +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 19:59:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418144 41.499840 41.381260 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.642578 107.628906 +util 63.250000 72.000000 59.000000 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 315 : avg iteration time 160.77 seconds +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 315 : data loading time 0.00 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 315 : avg forward pass time 71.74 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 315 : backward pass time 8.43 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 315 : optimizer step time 0.40 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 315-------------------- +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6338 iter count: 1.0 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6338 iter count: 1.0 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6338iter count: 1.0 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:02:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418172 41.499851 41.381283 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.539062 67.316406 +util 93.250000 100.000000 90.000000 +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 316 : avg iteration time 47.19 seconds +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 316 : data loading time 0.00 +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 316 : avg forward pass time 12.67 +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 316 : backward pass time 10.14 +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 316 : optimizer step time 1.56 +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 316-------------------- +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:03:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418375 41.500580 41.381325 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.406250 67.675781 67.316406 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 317 : avg iteration time 43.33 seconds +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 317 : data loading time 0.00 +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 317 : avg forward pass time 10.74 +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 317 : backward pass time 10.13 +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 317 : optimizer step time 1.58 +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 317-------------------- +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:04:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418370 41.500580 41.381283 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.435547 67.675781 67.355469 +util 97.000000 99.000000 94.000000 +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 318 : avg iteration time 46.16 seconds +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 318 : data loading time 0.00 +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 318 : avg forward pass time 12.13 +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 318 : backward pass time 10.14 +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 318 : optimizer step time 1.58 +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 318-------------------- +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:04:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418124 41.499611 41.381283 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.539062 67.316406 +util 94.500000 96.000000 91.000000 +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 319 : avg iteration time 43.37 seconds +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 319 : data loading time 0.00 +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 319 : avg forward pass time 10.77 +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 319 : backward pass time 10.15 +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 319 : optimizer step time 1.56 +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 319-------------------- +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:05:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417918 41.498775 41.381283 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.648438 107.564453 +util 62.750000 69.000000 55.000000 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 320 : avg iteration time 163.51 seconds +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 320 : data loading time 0.00 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 320 : avg forward pass time 73.12 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 320 : backward pass time 8.42 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 320 : optimizer step time 0.40 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 320-------------------- +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6533 iter count: 1.0 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6533 iter count: 1.0 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6533 iter count: 1.0 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6533iter count: 1.0 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:08:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417927 41.498775 41.381336 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.515137 67.988281 67.357422 +util 97.500000 99.000000 94.000000 +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 321 : avg iteration time 44.53 seconds +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 321 : data loading time 0.00 +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 321 : avg forward pass time 11.33 +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 321 : backward pass time 10.14 +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 321 : optimizer step time 1.57 +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 321-------------------- +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:09:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.417931 41.498775 41.381306 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.441895 67.753906 67.337891 +util 98.750000 99.000000 98.000000 +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 322 : avg iteration time 48.64 seconds +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 322 : data loading time 0.00 +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 322 : avg forward pass time 13.40 +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 322 : backward pass time 10.12 +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 322 : optimizer step time 1.56 +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 322-------------------- +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:09:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418249 41.500027 41.381298 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.476074 67.949219 67.318359 +util 94.000000 96.000000 90.000000 +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 323 : avg iteration time 48.72 seconds +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 323 : data loading time 0.00 +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 323 : avg forward pass time 13.43 +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 323 : backward pass time 10.15 +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 323 : optimizer step time 1.56 +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 323-------------------- +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:10:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418245 41.500027 41.381306 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.515137 67.988281 67.357422 +util 90.750000 99.000000 87.000000 +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 324 : avg iteration time 43.52 seconds +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 324 : data loading time 0.00 +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 324 : avg forward pass time 10.82 +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 324 : backward pass time 10.14 +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 324 : optimizer step time 1.57 +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 324-------------------- +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:11:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418302 41.500278 41.381290 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.648438 107.544922 +util 64.750000 74.000000 60.000000 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 325 : avg iteration time 158.46 seconds +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 325 : data loading time 0.00 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 325 : avg forward pass time 70.60 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 325 : backward pass time 8.47 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 325 : optimizer step time 0.40 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 325-------------------- +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5801 iter count: 1.0 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5801 iter count: 1.0 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5801 iter count: 1.0 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5801iter count: 1.0 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:14:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418108 41.499470 41.381336 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.713867 67.804688 67.441406 +util 96.500000 98.000000 93.000000 +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 326 : avg iteration time 47.25 seconds +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 326 : data loading time 0.00 +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 326 : avg forward pass time 12.68 +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 326 : backward pass time 10.16 +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 326 : optimizer step time 1.57 +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 326-------------------- +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:14:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418114 41.499470 41.381290 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.743164 67.843750 67.441406 +util 98.500000 100.000000 96.000000 +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 327 : avg iteration time 48.64 seconds +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 327 : data loading time 0.00 +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 327 : avg forward pass time 13.38 +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 327 : backward pass time 10.17 +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 327 : optimizer step time 1.56 +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 327-------------------- +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:15:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418185 41.499771 41.381294 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.679688 67.707031 67.597656 +util 93.250000 100.000000 90.000000 +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 328 : avg iteration time 48.68 seconds +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 328 : data loading time 0.00 +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 328 : avg forward pass time 13.41 +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 328 : backward pass time 10.15 +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 328 : optimizer step time 1.58 +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 328-------------------- +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:16:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418185 41.499771 41.381294 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.801758 67.882812 67.558594 +util 93.000000 100.000000 90.000000 +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 329 : avg iteration time 46.20 seconds +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 329 : data loading time 0.00 +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 329 : avg forward pass time 12.13 +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 329 : backward pass time 10.16 +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 329 : optimizer step time 1.57 +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 329-------------------- +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:17:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418244 41.500031 41.381294 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.648438 107.564453 +util 62.750000 65.000000 59.000000 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 330 : avg iteration time 166.06 seconds +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 330 : data loading time 0.00 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 330 : avg forward pass time 74.45 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 330 : backward pass time 8.41 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 330 : optimizer step time 0.40 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 330-------------------- +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6299 iter count: 1.0 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6299 iter count: 1.0 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6299 iter count: 1.0 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6299iter count: 1.0 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:20:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418134 41.499542 41.381340 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.480469 67.376953 +util 98.500000 100.000000 96.000000 +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 331 : avg iteration time 47.26 seconds +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 331 : data loading time 0.00 +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 331 : avg forward pass time 12.69 +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 331 : backward pass time 10.15 +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 331 : optimizer step time 1.57 +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 331-------------------- +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:20:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418123 41.499523 41.381317 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.729980 67.826172 67.441406 +util 98.500000 100.000000 96.000000 +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 332 : avg iteration time 48.79 seconds +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 332 : data loading time 0.00 +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 332 : avg forward pass time 13.42 +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 332 : backward pass time 10.13 +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 332 : optimizer step time 1.58 +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 332-------------------- +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:21:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418191 41.499775 41.381313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.432129 67.480469 67.416016 +util 91.250000 93.000000 89.000000 +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 333 : avg iteration time 43.50 seconds +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 333 : data loading time 0.00 +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 333 : avg forward pass time 10.82 +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 333 : backward pass time 10.15 +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 333 : optimizer step time 1.56 +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 333-------------------- +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:22:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418185 41.499775 41.381313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.441406 67.396484 +util 93.500000 100.000000 87.000000 +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 334 : avg iteration time 48.74 seconds +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 334 : data loading time 0.00 +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 334 : avg forward pass time 13.40 +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 334 : backward pass time 10.16 +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 334 : optimizer step time 1.57 +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 334-------------------- +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:23:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418194 41.499775 41.381317 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.687500 107.544922 +util 66.250000 68.000000 63.000000 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 335 : avg iteration time 163.56 seconds +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 335 : data loading time 0.00 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 335 : avg forward pass time 73.08 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 335 : backward pass time 8.48 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 335 : optimizer step time 0.41 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 335-------------------- +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6377 iter count: 1.0 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6377 iter count: 1.0 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6377 iter count: 1.0 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6377iter count: 1.0 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:25:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418250 41.500027 41.381313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.686035 67.708984 67.617188 +util 91.250000 99.000000 88.000000 +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 336 : avg iteration time 47.27 seconds +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 336 : data loading time 0.00 +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 336 : avg forward pass time 12.70 +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 336 : backward pass time 10.15 +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 336 : optimizer step time 1.58 +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 336-------------------- +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:26:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418255 41.500027 41.381332 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.734863 67.787109 67.578125 +util 99.000000 100.000000 98.000000 +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 337 : avg iteration time 43.56 seconds +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 337 : data loading time 0.00 +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 337 : avg forward pass time 10.84 +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 337 : backward pass time 10.15 +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 337 : optimizer step time 1.57 +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 337-------------------- +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:27:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418256 41.500027 41.381313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.728516 67.597656 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 338 : avg iteration time 48.80 seconds +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 338 : data loading time 0.00 +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 338 : avg forward pass time 13.47 +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 338 : backward pass time 10.14 +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 338 : optimizer step time 1.58 +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 338-------------------- +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:28:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418250 41.500027 41.381313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.739746 67.787109 67.597656 +util 93.750000 100.000000 91.000000 +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 339 : avg iteration time 46.07 seconds +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 339 : data loading time 0.00 +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 339 : avg forward pass time 12.11 +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 339 : backward pass time 10.14 +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 339 : optimizer step time 1.58 +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 339-------------------- +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:29:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418270 41.500065 41.381332 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.941895 108.052734 107.609375 +util 67.750000 70.000000 64.000000 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 340 : avg iteration time 158.11 seconds +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 340 : data loading time 0.00 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 340 : avg forward pass time 70.39 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 340 : backward pass time 8.42 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 340 : optimizer step time 0.40 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 340-------------------- +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6396 iter count: 1.0 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6396 iter count: 1.0 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6396 iter count: 1.0 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6396iter count: 1.0 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:31:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418267 41.500065 41.381317 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.700684 67.728516 67.617188 +util 94.250000 100.000000 91.000000 +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 341 : avg iteration time 44.69 seconds +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 341 : data loading time 0.00 +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 341 : avg forward pass time 11.43 +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 341 : backward pass time 10.14 +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 341 : optimizer step time 1.56 +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 341-------------------- +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:32:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418267 41.500065 41.381313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.661621 67.689453 67.578125 +util 95.750000 98.000000 92.000000 +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 342 : avg iteration time 43.46 seconds +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 342 : data loading time 0.00 +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 342 : avg forward pass time 10.79 +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 342 : backward pass time 10.14 +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 342 : optimizer step time 1.57 +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 342-------------------- +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:33:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418272 41.500065 41.381332 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.661621 67.689453 67.578125 +util 99.000000 100.000000 97.000000 +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 343 : avg iteration time 46.06 seconds +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 343 : data loading time 0.00 +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 343 : avg forward pass time 12.10 +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 343 : backward pass time 10.15 +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 343 : optimizer step time 1.57 +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 343-------------------- +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:33:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418340 41.500355 41.381313 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.705566 67.753906 67.689453 +util 93.250000 99.000000 90.000000 +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 344 : avg iteration time 48.67 seconds +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 344 : data loading time 0.00 +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 344 : avg forward pass time 13.41 +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 344 : backward pass time 10.15 +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 344 : optimizer step time 1.57 +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 344-------------------- +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:34:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:37:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418134 41.499523 41.381321 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.215332 108.365234 107.765625 +util 66.500000 69.000000 63.000000 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 345 : avg iteration time 158.69 seconds +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 345 : data loading time 0.00 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 345 : avg forward pass time 70.75 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 345 : backward pass time 8.42 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 345 : optimizer step time 0.40 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 345-------------------- +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6152 iter count: 1.0 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6152 iter count: 1.0 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6152 iter count: 1.0 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6152iter count: 1.0 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:37:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418144 41.499523 41.381321 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.637207 67.689453 67.480469 +util 94.000000 100.000000 91.000000 +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 346 : avg iteration time 49.91 seconds +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 346 : data loading time 0.00 +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 346 : avg forward pass time 14.00 +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 346 : backward pass time 10.16 +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 346 : optimizer step time 1.57 +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 346-------------------- +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:38:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418139 41.499523 41.381332 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.593262 67.689453 67.304688 +util 96.250000 98.000000 92.000000 +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 347 : avg iteration time 46.07 seconds +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 347 : data loading time 0.00 +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 347 : avg forward pass time 12.10 +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 347 : backward pass time 10.14 +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 347 : optimizer step time 1.56 +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 347-------------------- +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:38:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418134 41.499523 41.381321 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.656738 67.708984 67.500000 +util 98.250000 100.000000 95.000000 +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 348 : avg iteration time 48.83 seconds +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 348 : data loading time 0.00 +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 348 : avg forward pass time 13.44 +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 348 : backward pass time 10.24 +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 348 : optimizer step time 1.57 +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 348-------------------- +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:39:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418324 41.500271 41.381321 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.646973 67.708984 67.460938 +util 93.000000 100.000000 90.000000 +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 349 : avg iteration time 43.48 seconds +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 349 : data loading time 0.00 +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 349 : avg forward pass time 10.82 +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 349 : backward pass time 10.14 +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 349 : optimizer step time 1.57 +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 349-------------------- +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:40:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418276 41.500076 41.381321 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.623047 107.609375 +util 64.750000 73.000000 61.000000 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 350 : avg iteration time 158.24 seconds +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 350 : data loading time 0.00 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 350 : avg forward pass time 70.52 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 350 : backward pass time 8.42 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 350 : optimizer step time 0.40 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 350-------------------- +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6514 iter count: 1.0 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6514 iter count: 1.0 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6514 iter count: 1.0 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6514iter count: 1.0 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:43:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418304 41.500076 41.381321 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.705566 67.748047 67.578125 +util 91.500000 94.000000 87.000000 +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 351 : avg iteration time 47.19 seconds +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 351 : data loading time 0.00 +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 351 : avg forward pass time 12.66 +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 351 : backward pass time 10.14 +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 351 : optimizer step time 1.57 +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 351-------------------- +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:43:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418307 41.500084 41.381325 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.646973 67.767578 67.285156 +util 91.750000 99.000000 88.000000 +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 352 : avg iteration time 45.96 seconds +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 352 : data loading time 0.00 +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 352 : avg forward pass time 12.01 +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 352 : backward pass time 10.20 +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 352 : optimizer step time 1.57 +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 352-------------------- +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:44:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418313 41.500084 41.381325 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.617676 67.728516 67.285156 +util 91.750000 99.000000 88.000000 +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 353 : avg iteration time 48.80 seconds +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 353 : data loading time 0.00 +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 353 : avg forward pass time 13.47 +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 353 : backward pass time 10.14 +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 353 : optimizer step time 1.57 +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 353-------------------- +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:45:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418311 41.500084 41.381325 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.627441 67.728516 67.324219 +util 99.250000 100.000000 98.000000 +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 354 : avg iteration time 43.58 seconds +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 354 : data loading time 0.00 +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 354 : avg forward pass time 10.84 +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 354 : backward pass time 10.15 +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 354 : optimizer step time 1.59 +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 354-------------------- +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:46:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418235 41.499763 41.381325 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.765625 107.564453 +util 64.500000 72.000000 61.000000 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 355 : avg iteration time 160.66 seconds +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 355 : data loading time 0.00 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 355 : avg forward pass time 71.70 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 355 : backward pass time 8.44 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 355 : optimizer step time 0.40 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 355-------------------- +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5674 iter count: 1.0 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5674 iter count: 1.0 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5674 iter count: 1.0 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5674iter count: 1.0 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:48:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418231 41.499763 41.381329 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.601562 67.648438 67.460938 +util 95.250000 97.000000 91.000000 +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 356 : avg iteration time 47.24 seconds +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 356 : data loading time 0.00 +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 356 : avg forward pass time 12.71 +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 356 : backward pass time 10.13 +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 356 : optimizer step time 1.57 +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 356-------------------- +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:49:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418295 41.500019 41.381329 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.586914 67.628906 67.460938 +util 92.750000 95.000000 89.000000 +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 357 : avg iteration time 43.57 seconds +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 357 : data loading time 0.00 +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 357 : avg forward pass time 10.83 +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 357 : backward pass time 10.14 +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 357 : optimizer step time 1.58 +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 357-------------------- +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:50:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418308 41.500027 41.381332 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.713867 67.804688 67.441406 +util 91.250000 99.000000 88.000000 +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 358 : avg iteration time 43.55 seconds +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 358 : data loading time 0.00 +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 358 : avg forward pass time 10.81 +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 358 : backward pass time 10.16 +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 358 : optimizer step time 1.59 +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 358-------------------- +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:51:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418309 41.500015 41.381348 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.645508 67.707031 67.460938 +util 93.000000 95.000000 89.000000 +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 359 : avg iteration time 43.45 seconds +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 359 : data loading time 0.00 +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 359 : avg forward pass time 10.81 +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 359 : backward pass time 10.13 +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 359 : optimizer step time 1.57 +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 359-------------------- +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:51:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418010 41.498810 41.381348 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.546387 107.609375 107.525391 +util 69.750000 72.000000 66.000000 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 360 : avg iteration time 160.69 seconds +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 360 : data loading time 0.00 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 360 : avg forward pass time 71.73 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 360 : backward pass time 8.48 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 360 : optimizer step time 0.40 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 360-------------------- +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5781 iter count: 1.0 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5781 iter count: 1.0 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5781 iter count: 1.0 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5781iter count: 1.0 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:54:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418010 41.498810 41.381348 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.583496 67.728516 67.148438 +util 93.750000 100.000000 91.000000 +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 361 : avg iteration time 47.08 seconds +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 361 : data loading time 0.00 +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 361 : avg forward pass time 12.61 +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 361 : backward pass time 10.15 +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 361 : optimizer step time 1.58 +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 361-------------------- +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:55:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418079 41.499062 41.381359 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.568848 67.708984 67.148438 +util 98.500000 99.000000 97.000000 +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 362 : avg iteration time 43.33 seconds +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 362 : data loading time 0.00 +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 362 : avg forward pass time 10.74 +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 362 : backward pass time 10.13 +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 362 : optimizer step time 1.58 +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 362-------------------- +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:56:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418453 41.500565 41.381367 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.583496 67.728516 67.148438 +util 98.750000 99.000000 98.000000 +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 363 : avg iteration time 46.41 seconds +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 363 : data loading time 0.00 +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 363 : avg forward pass time 12.24 +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 363 : backward pass time 10.14 +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 363 : optimizer step time 1.58 +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 363-------------------- +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:56:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418506 41.500793 41.381348 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.583496 67.728516 67.148438 +util 96.000000 100.000000 87.000000 +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 364 : avg iteration time 43.39 seconds +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 364 : data loading time 0.00 +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 364 : avg forward pass time 10.77 +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 364 : backward pass time 10.15 +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 364 : optimizer step time 1.57 +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 364-------------------- +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 20:57:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418093 41.499100 41.381363 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.628906 107.603516 +util 62.750000 65.000000 58.000000 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 365 : avg iteration time 160.71 seconds +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 365 : data loading time 0.00 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 365 : avg forward pass time 71.76 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 365 : backward pass time 8.38 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 365 : optimizer step time 0.40 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 365-------------------- +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6299 iter count: 1.0 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6299 iter count: 1.0 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6299 iter count: 1.0 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6299iter count: 1.0 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:00:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418090 41.499104 41.381348 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.616211 67.707031 67.343750 +util 93.500000 100.000000 90.000000 +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 366 : avg iteration time 47.15 seconds +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 366 : data loading time 0.00 +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 366 : avg forward pass time 12.62 +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 366 : backward pass time 10.16 +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 366 : optimizer step time 1.57 +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 366-------------------- +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:01:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418093 41.499104 41.381348 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.689453 67.804688 67.343750 +util 98.500000 100.000000 96.000000 +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 367 : avg iteration time 46.12 seconds +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 367 : data loading time 0.00 +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 367 : avg forward pass time 12.11 +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 367 : backward pass time 10.16 +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 367 : optimizer step time 1.58 +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 367-------------------- +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:01:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418532 41.500862 41.381348 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.625977 67.707031 67.382812 +util 91.500000 99.000000 88.000000 +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 368 : avg iteration time 48.86 seconds +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 368 : data loading time 0.00 +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 368 : avg forward pass time 13.49 +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 368 : backward pass time 10.15 +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 368 : optimizer step time 1.56 +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 368-------------------- +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:02:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418595 41.501114 41.381348 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.655273 67.687500 67.558594 +util 99.000000 100.000000 97.000000 +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 369 : avg iteration time 48.76 seconds +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 369 : data loading time 0.00 +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 369 : avg forward pass time 13.45 +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 369 : backward pass time 10.14 +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 369 : optimizer step time 1.56 +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 369-------------------- +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:03:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418556 41.500938 41.381348 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.628906 107.564453 +util 63.750000 72.000000 59.000000 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 370 : avg iteration time 163.68 seconds +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 370 : data loading time 0.00 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 370 : avg forward pass time 73.27 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 370 : backward pass time 8.39 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 370 : optimizer step time 0.40 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 370-------------------- +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6016 iter count: 1.0 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6016 iter count: 1.0 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6016 iter count: 1.0 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6016iter count: 1.0 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:06:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418380 41.500233 41.381359 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.656738 67.787109 67.265625 +util 97.500000 99.000000 94.000000 +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 371 : avg iteration time 44.68 seconds +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 371 : data loading time 0.00 +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 371 : avg forward pass time 11.41 +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 371 : backward pass time 10.13 +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 371 : optimizer step time 1.57 +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 371-------------------- +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:06:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418384 41.500256 41.381351 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.666504 67.708984 67.539062 +util 99.000000 100.000000 97.000000 +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 372 : avg iteration time 43.44 seconds +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 372 : data loading time 0.00 +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 372 : avg forward pass time 10.78 +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 372 : backward pass time 10.16 +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 372 : optimizer step time 1.57 +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 372-------------------- +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:07:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418447 41.500504 41.381351 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.593262 67.708984 67.246094 +util 92.500000 95.000000 88.000000 +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 373 : avg iteration time 48.69 seconds +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 373 : data loading time 0.00 +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 373 : avg forward pass time 13.39 +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 373 : backward pass time 10.16 +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 373 : optimizer step time 1.58 +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 373-------------------- +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:08:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418440 41.500481 41.381351 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.593262 67.708984 67.246094 +util 94.000000 100.000000 91.000000 +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 374 : avg iteration time 46.18 seconds +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 374 : data loading time 0.00 +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 374 : avg forward pass time 12.15 +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 374 : backward pass time 10.14 +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 374 : optimizer step time 1.57 +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 374-------------------- +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:09:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418447 41.500507 41.381351 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.707520 107.740234 107.609375 +util 65.750000 74.000000 61.000000 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 375 : avg iteration time 160.66 seconds +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 375 : data loading time 0.00 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 375 : avg forward pass time 71.71 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 375 : backward pass time 8.37 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 375 : optimizer step time 0.43 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 375-------------------- +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5908 iter count: 1.0 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5908 iter count: 1.0 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5908 iter count: 1.0 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5908iter count: 1.0 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:11:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418450 41.500492 41.381363 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.666504 67.728516 67.480469 +util 91.500000 99.000000 88.000000 +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 376 : avg iteration time 47.10 seconds +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 376 : data loading time 0.00 +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 376 : avg forward pass time 12.63 +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 376 : backward pass time 10.15 +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 376 : optimizer step time 1.57 +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 376-------------------- +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:12:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418463 41.500515 41.381371 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.651855 67.708984 67.480469 +util 93.000000 99.000000 90.000000 +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 377 : avg iteration time 45.83 seconds +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 377 : data loading time 0.00 +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 377 : avg forward pass time 11.99 +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 377 : backward pass time 10.15 +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 377 : optimizer step time 1.57 +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 377-------------------- +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:13:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418455 41.500492 41.381363 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.666504 67.787109 67.304688 +util 94.250000 96.000000 90.000000 +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 378 : avg iteration time 45.85 seconds +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 378 : data loading time 0.00 +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 378 : avg forward pass time 12.00 +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 378 : backward pass time 10.15 +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 378 : optimizer step time 1.57 +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 378-------------------- +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:14:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418456 41.500492 41.381363 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.617676 67.728516 67.285156 +util 92.000000 100.000000 88.000000 +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 379 : avg iteration time 45.92 seconds +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 379 : data loading time 0.00 +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 379 : avg forward pass time 12.03 +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 379 : backward pass time 10.16 +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 379 : optimizer step time 1.58 +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 379-------------------- +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:14:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418460 41.500492 41.381363 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.603516 107.570312 +util 63.000000 71.000000 59.000000 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 380 : avg iteration time 160.22 seconds +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 380 : data loading time 0.00 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 380 : avg forward pass time 71.54 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 380 : backward pass time 8.38 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 380 : optimizer step time 0.39 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 380-------------------- +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6074 iter count: 1.0 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6074 iter count: 1.0 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6074 iter count: 1.0 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6074iter count: 1.0 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:17:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418458 41.500500 41.381363 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.333008 67.335938 67.324219 +util 92.000000 100.000000 88.000000 +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 381 : avg iteration time 46.97 seconds +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 381 : data loading time 0.00 +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 381 : avg forward pass time 12.55 +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 381 : backward pass time 10.15 +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 381 : optimizer step time 1.57 +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 381-------------------- +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:18:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418465 41.500523 41.381367 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.414062 67.363281 +util 98.250000 100.000000 95.000000 +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 382 : avg iteration time 48.53 seconds +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 382 : data loading time 0.00 +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 382 : avg forward pass time 13.32 +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 382 : backward pass time 10.15 +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 382 : optimizer step time 1.58 +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 382-------------------- +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:19:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418461 41.500504 41.381371 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.342773 67.480469 67.296875 +util 94.500000 100.000000 91.000000 +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 383 : avg iteration time 43.31 seconds +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 383 : data loading time 0.00 +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 383 : avg forward pass time 10.72 +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 383 : backward pass time 10.14 +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 383 : optimizer step time 1.58 +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 383-------------------- +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:19:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418564 41.500904 41.381363 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.367188 67.375000 67.343750 +util 97.250000 99.000000 93.000000 +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 384 : avg iteration time 48.48 seconds +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 384 : data loading time 0.00 +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 384 : avg forward pass time 13.30 +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 384 : backward pass time 10.15 +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 384 : optimizer step time 1.57 +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 384-------------------- +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:20:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418797 41.501842 41.381374 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.692871 107.701172 107.667969 +util 69.750000 72.000000 66.000000 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 385 : avg iteration time 160.24 seconds +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 385 : data loading time 0.00 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 385 : avg forward pass time 71.54 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 385 : backward pass time 8.42 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 385 : optimizer step time 0.39 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 385-------------------- +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5928 iter count: 1.0 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5928 iter count: 1.0 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5928 iter count: 1.0 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5928iter count: 1.0 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:23:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418807 41.501873 41.381386 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.772461 67.882812 67.441406 +util 98.000000 99.000000 95.000000 +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 386 : avg iteration time 49.58 seconds +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 386 : data loading time 0.00 +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 386 : avg forward pass time 13.85 +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 386 : backward pass time 10.15 +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 386 : optimizer step time 1.57 +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 386-------------------- +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:24:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418810 41.501896 41.381374 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.684570 67.785156 67.382812 +util 93.500000 100.000000 90.000000 +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 387 : avg iteration time 45.89 seconds +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 387 : data loading time 0.00 +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 387 : avg forward pass time 12.01 +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 387 : backward pass time 10.15 +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 387 : optimizer step time 1.56 +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 387-------------------- +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:25:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418848 41.502026 41.381397 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.772461 67.882812 67.441406 +util 92.750000 95.000000 88.000000 +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 388 : avg iteration time 45.83 seconds +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 388 : data loading time 0.00 +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 388 : avg forward pass time 11.99 +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 388 : backward pass time 10.12 +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 388 : optimizer step time 1.57 +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 388-------------------- +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:25:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419016 41.502716 41.381378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 91.000000 99.000000 87.000000 +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 389 : avg iteration time 48.53 seconds +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 389 : data loading time 0.00 +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 389 : avg forward pass time 13.32 +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 389 : backward pass time 10.15 +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 389 : optimizer step time 1.58 +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 389-------------------- +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:26:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418694 41.501431 41.381378 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.590332 107.785156 107.525391 +util 70.000000 72.000000 66.000000 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 390 : avg iteration time 165.37 seconds +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 390 : data loading time 0.00 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 390 : avg forward pass time 74.13 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 390 : backward pass time 8.38 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 390 : optimizer step time 0.39 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 390-------------------- +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6309 iter count: 1.0 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6309 iter count: 1.0 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6309 iter count: 1.0 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6309iter count: 1.0 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:29:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418694 41.501431 41.381378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.196289 67.218750 67.128906 +util 94.750000 97.000000 90.000000 +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 391 : avg iteration time 46.92 seconds +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 391 : data loading time 0.00 +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 391 : avg forward pass time 12.52 +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 391 : backward pass time 10.14 +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 391 : optimizer step time 1.58 +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 391-------------------- +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:30:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418723 41.501545 41.381378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.284180 67.335938 67.128906 +util 90.000000 98.000000 86.000000 +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 392 : avg iteration time 45.90 seconds +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 392 : data loading time 0.00 +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 392 : avg forward pass time 12.02 +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 392 : backward pass time 10.14 +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 392 : optimizer step time 1.57 +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 392-------------------- +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:30:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418730 41.501537 41.381378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.318359 67.375000 67.148438 +util 93.500000 96.000000 89.000000 +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 393 : avg iteration time 45.89 seconds +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 393 : data loading time 0.00 +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 393 : avg forward pass time 12.00 +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 393 : backward pass time 10.15 +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 393 : optimizer step time 1.57 +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 393-------------------- +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:31:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418732 41.501556 41.381378 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.250000 67.296875 67.109375 +util 96.500000 99.000000 92.000000 +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 394 : avg iteration time 43.23 seconds +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 394 : data loading time 0.00 +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 394 : avg forward pass time 10.68 +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 394 : backward pass time 10.16 +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 394 : optimizer step time 1.57 +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 394-------------------- +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:32:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418653 41.501263 41.381382 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.746094 107.544922 +util 68.000000 71.000000 63.000000 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 395 : avg iteration time 154.84 seconds +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 395 : data loading time 0.00 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 395 : avg forward pass time 68.82 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 395 : backward pass time 8.43 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 395 : optimizer step time 0.40 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 395-------------------- +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5605 iter count: 1.0 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5605 iter count: 1.0 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5605 iter count: 1.0 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5605iter count: 1.0 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:34:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418688 41.501259 41.381382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.406250 67.675781 67.316406 +util 94.750000 100.000000 92.000000 +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 396 : avg iteration time 44.30 seconds +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 396 : data loading time 0.00 +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 396 : avg forward pass time 11.23 +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 396 : backward pass time 10.14 +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 396 : optimizer step time 1.57 +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 396-------------------- +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:35:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418710 41.501369 41.381382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.519531 67.375000 +util 95.000000 100.000000 92.000000 +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 397 : avg iteration time 46.40 seconds +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 397 : data loading time 0.00 +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 397 : avg forward pass time 12.28 +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 397 : backward pass time 10.12 +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 397 : optimizer step time 1.57 +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 397-------------------- +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:36:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418715 41.501369 41.381382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.401367 67.539062 67.355469 +util 98.250000 99.000000 96.000000 +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 398 : avg iteration time 48.52 seconds +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 398 : data loading time 0.00 +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 398 : avg forward pass time 13.32 +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 398 : backward pass time 10.15 +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 398 : optimizer step time 1.56 +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 398-------------------- +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:37:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418746 41.501499 41.381382 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.323242 67.402344 67.296875 +util 90.500000 98.000000 88.000000 +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 399 : avg iteration time 45.82 seconds +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 399 : data loading time 0.00 +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 399 : avg forward pass time 12.00 +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 399 : backward pass time 10.12 +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 399 : optimizer step time 1.56 +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 399-------------------- +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:38:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418484 41.500465 41.381382 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.668457 107.681641 107.628906 +util 63.500000 71.000000 59.000000 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 400 : avg iteration time 154.87 seconds +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 400 : data loading time 0.00 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 400 : avg forward pass time 68.85 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 400 : backward pass time 8.39 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 400 : optimizer step time 0.41 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 400-------------------- +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5879 iter count: 1.0 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5879 iter count: 1.0 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5879 iter count: 1.0 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5879iter count: 1.0 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 21:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 21:40:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.27 seconds +[May 4, 2026 - 21:40:50 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 400 +[May 4, 2026 - 21:40:50 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000400 +[May 4, 2026 - 21:41:08 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 400 +[May 4, 2026 - 21:41:08 | INFO | fastgen.trainer:validate:358 ] Validation iteration 400 +[May 4, 2026 - 21:41:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 4, 2026 - 21:41:14 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.405 +[May 4, 2026 - 21:41:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 21:41:19 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.823 +[May 4, 2026 - 21:41:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 21:41:24 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.506 +[May 4, 2026 - 21:41:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 21:41:29 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.182 +[May 4, 2026 - 21:41:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 21:41:34 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.760 +[May 4, 2026 - 21:41:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 21:41:39 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.774 +[May 4, 2026 - 21:41:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 21:41:44 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.488 +[May 4, 2026 - 21:41:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 21:41:49 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.218 +[May 4, 2026 - 21:41:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 21:41:54 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.246 +[May 4, 2026 - 21:41:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 21:41:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 21:41:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.902 +[May 4, 2026 - 21:41:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 400-------------------- +[May 4, 2026 - 21:41:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 21:42:15 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 400 +[May 4, 2026 - 21:42:15 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.530 (n=10) +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419981 41.506054 41.381527 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.333984 68.574219 68.253906 +util 90.000000 98.000000 86.000000 +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 401 : avg iteration time 144.13 seconds +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 401 : data loading time 0.00 +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 401 : avg forward pass time 11.66 +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 401 : backward pass time 10.15 +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 401 : optimizer step time 1.57 +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 401-------------------- +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:43:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419984 41.506054 41.381527 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.329102 68.554688 68.253906 +util 98.500000 100.000000 96.000000 +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 402 : avg iteration time 43.37 seconds +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 402 : data loading time 0.00 +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 402 : avg forward pass time 10.75 +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 402 : backward pass time 10.16 +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 402 : optimizer step time 1.56 +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 402-------------------- +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:43:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419937 41.505890 41.381527 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.324219 68.535156 68.253906 +util 92.500000 100.000000 89.000000 +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 403 : avg iteration time 46.06 seconds +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 403 : data loading time 0.00 +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 403 : avg forward pass time 12.07 +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 403 : backward pass time 10.16 +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 403 : optimizer step time 1.59 +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 403-------------------- +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:44:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419946 41.505901 41.381527 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.343750 68.554688 68.273438 +util 91.000000 99.000000 87.000000 +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 404 : avg iteration time 45.98 seconds +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 404 : data loading time 0.00 +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 404 : avg forward pass time 12.05 +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 404 : backward pass time 10.15 +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 404 : optimizer step time 1.57 +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 404-------------------- +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:45:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418897 41.501682 41.381550 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.678223 107.726562 107.662109 +util 65.250000 68.000000 61.000000 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 405 : avg iteration time 162.72 seconds +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 405 : data loading time 0.00 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 405 : avg forward pass time 72.80 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 405 : backward pass time 8.38 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 405 : optimizer step time 0.40 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 405-------------------- +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5869 iter count: 1.0 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5869 iter count: 1.0 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5869 iter count: 1.0 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5869iter count: 1.0 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:47:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418895 41.501682 41.381527 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.843750 67.519531 +util 97.250000 99.000000 93.000000 +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 406 : avg iteration time 47.01 seconds +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 406 : data loading time 0.00 +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 406 : avg forward pass time 12.58 +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 406 : backward pass time 10.14 +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 406 : optimizer step time 1.57 +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 406-------------------- +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:48:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418944 41.501877 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.699219 67.765625 67.500000 +util 91.000000 99.000000 87.000000 +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 407 : avg iteration time 45.87 seconds +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 407 : data loading time 0.00 +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 407 : avg forward pass time 12.01 +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 407 : backward pass time 10.13 +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 407 : optimizer step time 1.57 +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 407-------------------- +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:49:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419182 41.502834 41.381561 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.630859 67.667969 67.519531 +util 92.750000 95.000000 88.000000 +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 408 : avg iteration time 43.27 seconds +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 408 : data loading time 0.00 +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 408 : avg forward pass time 10.70 +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 408 : backward pass time 10.15 +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 408 : optimizer step time 1.57 +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 408-------------------- +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:50:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419250 41.503086 41.381561 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.630859 67.667969 67.519531 +util 95.000000 97.000000 90.000000 +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 409 : avg iteration time 45.83 seconds +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 409 : data loading time 0.00 +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 409 : avg forward pass time 12.00 +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 409 : backward pass time 10.12 +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 409 : optimizer step time 1.57 +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 409-------------------- +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:51:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419106 41.502487 41.381561 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.667969 107.603516 +util 60.250000 68.000000 56.000000 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 410 : avg iteration time 160.12 seconds +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 410 : data loading time 0.00 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 410 : avg forward pass time 71.49 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 410 : backward pass time 8.39 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 410 : optimizer step time 0.40 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 410-------------------- +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5664 iter count: 1.0 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5664 iter count: 1.0 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5664 iter count: 1.0 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5664iter count: 1.0 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:53:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418975 41.502029 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 98.000000 100.000000 94.000000 +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 411 : avg iteration time 49.73 seconds +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 411 : data loading time 0.00 +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 411 : avg forward pass time 13.94 +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 411 : backward pass time 10.13 +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 411 : optimizer step time 1.57 +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 411-------------------- +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:54:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418984 41.502029 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.723633 67.882812 67.246094 +util 92.500000 100.000000 88.000000 +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 412 : avg iteration time 45.87 seconds +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 412 : data loading time 0.00 +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 412 : avg forward pass time 12.00 +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 412 : backward pass time 10.15 +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 412 : optimizer step time 1.57 +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 412-------------------- +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:55:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419035 41.502239 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.723633 67.882812 67.246094 +util 96.750000 99.000000 92.000000 +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 413 : avg iteration time 45.90 seconds +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 413 : data loading time 0.00 +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 413 : avg forward pass time 12.01 +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 413 : backward pass time 10.14 +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 413 : optimizer step time 1.58 +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 413-------------------- +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:56:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419097 41.502476 41.381561 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.738281 67.902344 67.246094 +util 97.500000 99.000000 93.000000 +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 414 : avg iteration time 46.02 seconds +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 414 : data loading time 0.00 +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 414 : avg forward pass time 12.07 +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 414 : backward pass time 10.13 +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 414 : optimizer step time 1.58 +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 414-------------------- +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:56:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419154 41.502697 41.381561 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.580566 107.687500 107.544922 +util 64.250000 67.000000 60.000000 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 415 : avg iteration time 157.37 seconds +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 415 : data loading time 0.00 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 415 : avg forward pass time 70.11 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 415 : backward pass time 8.41 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 415 : optimizer step time 0.40 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 415-------------------- +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6133 iter count: 1.0 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6133 iter count: 1.0 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6133 iter count: 1.0 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6133iter count: 1.0 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 21:59:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419157 41.502697 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.728516 67.843750 67.382812 +util 98.250000 100.000000 95.000000 +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 416 : avg iteration time 46.84 seconds +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 416 : data loading time 0.00 +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 416 : avg forward pass time 12.50 +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 416 : backward pass time 10.13 +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 416 : optimizer step time 1.57 +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 416-------------------- +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:00:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419163 41.502712 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.796875 67.941406 67.363281 +util 97.750000 100.000000 93.000000 +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 417 : avg iteration time 43.18 seconds +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 417 : data loading time 0.00 +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 417 : avg forward pass time 10.69 +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 417 : backward pass time 10.12 +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 417 : optimizer step time 1.56 +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 417-------------------- +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:00:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419148 41.502697 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.882812 67.363281 +util 90.750000 98.000000 87.000000 +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 418 : avg iteration time 48.46 seconds +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 418 : data loading time 0.00 +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 418 : avg forward pass time 13.30 +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 418 : backward pass time 10.15 +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 418 : optimizer step time 1.58 +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 418-------------------- +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:01:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419262 41.503059 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 67.882812 67.343750 +util 98.000000 100.000000 94.000000 +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 419 : avg iteration time 48.55 seconds +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 419 : data loading time 0.00 +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 419 : avg forward pass time 13.34 +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 419 : backward pass time 10.14 +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 419 : optimizer step time 1.58 +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 419-------------------- +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:02:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419072 41.502289 41.381561 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.668457 107.687500 107.662109 +util 64.000000 67.000000 59.000000 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 420 : avg iteration time 162.76 seconds +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 420 : data loading time 0.00 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 420 : avg forward pass time 72.79 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 420 : backward pass time 8.42 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 420 : optimizer step time 0.40 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 420-------------------- +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6035 iter count: 1.0 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6035 iter count: 1.0 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6035 iter count: 1.0 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6035iter count: 1.0 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:05:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419072 41.502289 41.381561 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.699219 67.863281 67.207031 +util 93.750000 100.000000 90.000000 +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 421 : avg iteration time 49.54 seconds +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 421 : data loading time 0.00 +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 421 : avg forward pass time 13.85 +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 421 : backward pass time 10.13 +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 421 : optimizer step time 1.57 +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 421-------------------- +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:06:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419073 41.502289 41.381557 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.767578 67.902344 67.363281 +util 97.750000 99.000000 94.000000 +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 422 : avg iteration time 43.30 seconds +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 422 : data loading time 0.00 +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 422 : avg forward pass time 10.72 +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 422 : backward pass time 10.14 +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 422 : optimizer step time 1.57 +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 422-------------------- +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:06:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419076 41.502289 41.381557 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.882812 67.363281 +util 93.500000 99.000000 89.000000 +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 423 : avg iteration time 46.03 seconds +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 423 : data loading time 0.00 +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 423 : avg forward pass time 12.07 +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 423 : backward pass time 10.15 +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 423 : optimizer step time 1.59 +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 423-------------------- +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:07:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419379 41.503456 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.882812 67.363281 +util 92.500000 99.000000 89.000000 +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 424 : avg iteration time 45.84 seconds +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 424 : data loading time 0.00 +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 424 : avg forward pass time 12.00 +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 424 : backward pass time 10.13 +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 424 : optimizer step time 1.58 +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 424-------------------- +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:08:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419188 41.502651 41.381584 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.702637 107.720703 107.648438 +util 61.500000 69.000000 57.000000 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 425 : avg iteration time 155.02 seconds +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 425 : data loading time 0.00 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 425 : avg forward pass time 68.92 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 425 : backward pass time 8.39 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 425 : optimizer step time 0.41 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 425-------------------- +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5586 iter count: 1.0 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5586 iter count: 1.0 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5586 iter count: 1.0 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5586iter count: 1.0 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:10:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419172 41.502651 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.528320 67.570312 67.402344 +util 93.500000 96.000000 88.000000 +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 426 : avg iteration time 49.64 seconds +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 426 : data loading time 0.00 +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 426 : avg forward pass time 13.88 +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 426 : backward pass time 10.16 +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 426 : optimizer step time 1.57 +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 426-------------------- +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0152 iter count: 1.0 +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0152 iter count: 1.0 +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:11:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419175 41.502663 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.557617 67.589844 67.460938 +util 94.250000 100.000000 91.000000 +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 427 : avg iteration time 48.56 seconds +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 427 : data loading time 0.00 +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 427 : avg forward pass time 13.33 +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 427 : backward pass time 10.15 +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 427 : optimizer step time 1.57 +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 427-------------------- +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:12:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419180 41.502651 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.528320 67.570312 67.402344 +util 98.500000 100.000000 96.000000 +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 428 : avg iteration time 45.93 seconds +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 428 : data loading time 0.00 +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 428 : avg forward pass time 12.02 +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 428 : backward pass time 10.17 +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 428 : optimizer step time 1.57 +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 428-------------------- +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:13:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419182 41.502666 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.528320 67.570312 67.402344 +util 95.500000 99.000000 93.000000 +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 429 : avg iteration time 45.90 seconds +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 429 : data loading time 0.00 +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 429 : avg forward pass time 12.02 +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 429 : backward pass time 10.14 +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 429 : optimizer step time 1.57 +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 429-------------------- +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:14:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419104 41.502270 41.381538 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.712402 107.740234 107.628906 +util 65.000000 68.000000 60.000000 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 430 : avg iteration time 162.84 seconds +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 430 : data loading time 0.00 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 430 : avg forward pass time 72.85 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 430 : backward pass time 8.39 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 430 : optimizer step time 0.39 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 430-------------------- +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5938 iter count: 1.0 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5938 iter count: 1.0 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5938 iter count: 1.0 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5938iter count: 1.0 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:16:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419101 41.502270 41.381538 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.801758 68.019531 67.148438 +util 92.500000 100.000000 89.000000 +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 431 : avg iteration time 46.91 seconds +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 431 : data loading time 0.00 +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 431 : avg forward pass time 12.52 +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 431 : backward pass time 10.15 +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 431 : optimizer step time 1.56 +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 431-------------------- +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:17:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419166 41.502510 41.381550 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.882812 67.363281 +util 91.000000 99.000000 87.000000 +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 432 : avg iteration time 45.98 seconds +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 432 : data loading time 0.00 +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 432 : avg forward pass time 12.04 +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 432 : backward pass time 10.16 +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 432 : optimizer step time 1.58 +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 432-------------------- +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:18:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419169 41.502525 41.381550 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.694336 67.882812 67.128906 +util 95.000000 100.000000 92.000000 +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 433 : avg iteration time 43.26 seconds +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 433 : data loading time 0.00 +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 433 : avg forward pass time 10.70 +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 433 : backward pass time 10.15 +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 433 : optimizer step time 1.57 +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 433-------------------- +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:19:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419169 41.502510 41.381550 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.694336 67.882812 67.128906 +util 94.000000 99.000000 91.000000 +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 434 : avg iteration time 45.99 seconds +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 434 : data loading time 0.00 +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 434 : avg forward pass time 12.06 +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 434 : backward pass time 10.14 +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 434 : optimizer step time 1.56 +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 434-------------------- +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:19:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419000 41.501820 41.381561 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.623047 107.589844 +util 60.000000 68.000000 56.000000 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 435 : avg iteration time 160.18 seconds +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 435 : data loading time 0.00 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 435 : avg forward pass time 71.50 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 435 : backward pass time 8.42 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 435 : optimizer step time 0.40 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 435-------------------- +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5762 iter count: 1.0 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5762 iter count: 1.0 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5762 iter count: 1.0 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5762iter count: 1.0 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:22:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419003 41.501816 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.863281 67.421875 +util 98.000000 99.000000 95.000000 +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 436 : avg iteration time 46.95 seconds +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 436 : data loading time 0.00 +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 436 : avg forward pass time 12.54 +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 436 : backward pass time 10.15 +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 436 : optimizer step time 1.58 +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 436-------------------- +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:23:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419063 41.502060 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.882812 67.363281 +util 91.500000 99.000000 88.000000 +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 437 : avg iteration time 48.65 seconds +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 437 : data loading time 0.00 +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 437 : avg forward pass time 13.37 +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 437 : backward pass time 10.16 +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 437 : optimizer step time 1.57 +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 437-------------------- +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:24:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419069 41.502087 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.787109 67.902344 67.441406 +util 95.750000 98.000000 91.000000 +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 438 : avg iteration time 43.30 seconds +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 438 : data loading time 0.00 +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 438 : avg forward pass time 10.72 +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 438 : backward pass time 10.14 +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 438 : optimizer step time 1.57 +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 438-------------------- +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:24:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419069 41.502087 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777344 67.902344 67.402344 +util 93.500000 100.000000 90.000000 +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 439 : avg iteration time 45.93 seconds +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 439 : data loading time 0.00 +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 439 : avg forward pass time 12.04 +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 439 : backward pass time 10.14 +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 439 : optimizer step time 1.58 +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 439-------------------- +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:25:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418926 41.501514 41.381573 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.642578 107.628906 +util 69.000000 72.000000 64.000000 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 440 : avg iteration time 165.33 seconds +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 440 : data loading time 0.00 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 440 : avg forward pass time 74.09 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 440 : backward pass time 8.40 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 440 : optimizer step time 0.39 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 440-------------------- +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5977 iter count: 1.0 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5977 iter count: 1.0 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5977 iter count: 1.0 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5977iter count: 1.0 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:28:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418928 41.501514 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.480469 67.335938 +util 93.000000 99.000000 88.000000 +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 441 : avg iteration time 44.23 seconds +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 441 : data loading time 0.00 +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 441 : avg forward pass time 11.18 +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 441 : backward pass time 10.15 +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 441 : optimizer step time 1.56 +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 441-------------------- +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:29:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418992 41.501759 41.381592 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.259766 67.441406 67.199219 +util 93.000000 99.000000 88.000000 +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 442 : avg iteration time 46.00 seconds +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 442 : data loading time 0.00 +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 442 : avg forward pass time 12.08 +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 442 : backward pass time 10.13 +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 442 : optimizer step time 1.55 +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 442-------------------- +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:29:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419172 41.502491 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.713867 67.863281 67.265625 +util 93.250000 99.000000 90.000000 +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 443 : avg iteration time 43.25 seconds +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 443 : data loading time 0.00 +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 443 : avg forward pass time 10.69 +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 443 : backward pass time 10.13 +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 443 : optimizer step time 1.56 +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 443-------------------- +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:30:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419170 41.502491 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.381836 67.460938 67.355469 +util 97.250000 99.000000 93.000000 +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 444 : avg iteration time 45.90 seconds +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 444 : data loading time 0.00 +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 444 : avg forward pass time 12.02 +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 444 : backward pass time 10.13 +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 444 : optimizer step time 1.57 +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 444-------------------- +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:31:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418858 41.501232 41.381573 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.609863 107.628906 107.603516 +util 62.250000 69.000000 58.000000 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 445 : avg iteration time 162.91 seconds +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 445 : data loading time 0.00 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 445 : avg forward pass time 72.86 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 445 : backward pass time 8.40 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 445 : optimizer step time 0.40 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 445-------------------- +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6035 iter count: 1.0 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6035 iter count: 1.0 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6035 iter count: 1.0 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6035iter count: 1.0 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:34:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418859 41.501236 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.762695 67.882812 67.402344 +util 97.750000 100.000000 93.000000 +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 446 : avg iteration time 49.50 seconds +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 446 : data loading time 0.00 +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 446 : avg forward pass time 13.81 +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 446 : backward pass time 10.15 +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 446 : optimizer step time 1.57 +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 446-------------------- +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:34:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.418911 41.501442 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.757812 67.882812 67.382812 +util 98.500000 100.000000 95.000000 +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 447 : avg iteration time 45.82 seconds +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 447 : data loading time 0.00 +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 447 : avg forward pass time 11.98 +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 447 : backward pass time 10.13 +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 447 : optimizer step time 1.58 +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 447-------------------- +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:35:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419094 41.502174 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.738281 67.863281 67.363281 +util 98.000000 100.000000 94.000000 +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 448 : avg iteration time 48.60 seconds +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 448 : data loading time 0.00 +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 448 : avg forward pass time 13.37 +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 448 : backward pass time 10.14 +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 448 : optimizer step time 1.58 +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 448-------------------- +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:36:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419157 41.502426 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.772461 67.902344 67.382812 +util 92.250000 99.000000 89.000000 +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 449 : avg iteration time 45.86 seconds +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 449 : data loading time 0.00 +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 449 : avg forward pass time 11.99 +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 449 : backward pass time 10.15 +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 449 : optimizer step time 1.58 +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 449-------------------- +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:37:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419230 41.502708 41.381573 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.697754 107.740234 107.570312 +util 65.250000 68.000000 60.000000 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 450 : avg iteration time 160.30 seconds +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 450 : data loading time 0.00 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 450 : avg forward pass time 71.57 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 450 : backward pass time 8.42 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 450 : optimizer step time 0.39 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 450-------------------- +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6094 iter count: 1.0 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6094 iter count: 1.0 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6094 iter count: 1.0 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6094iter count: 1.0 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:39:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419079 41.502102 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 67.863281 67.402344 +util 94.750000 99.000000 92.000000 +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 451 : avg iteration time 49.54 seconds +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 451 : data loading time 0.00 +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 451 : avg forward pass time 13.84 +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 451 : backward pass time 10.15 +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 451 : optimizer step time 1.57 +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 451-------------------- +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419077 41.502102 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.743164 67.863281 67.382812 +util 95.750000 99.000000 89.000000 +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 452 : avg iteration time 45.92 seconds +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 452 : data loading time 0.00 +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 452 : avg forward pass time 12.02 +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 452 : backward pass time 10.15 +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 452 : optimizer step time 1.57 +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 452-------------------- +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:41:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.419077 41.502102 41.381573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.748047 67.863281 67.402344 +util 96.500000 99.000000 91.000000 +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 453 : avg iteration time 48.55 seconds +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 453 : data loading time 0.00 +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 453 : avg forward pass time 13.33 +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 453 : backward pass time 10.13 +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 453 : optimizer step time 1.58 +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 453-------------------- +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:42:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.608952 41.701248 41.569008 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.882812 67.363281 +util 93.000000 100.000000 89.000000 +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 454 : avg iteration time 48.51 seconds +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 454 : data loading time 0.00 +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 454 : avg forward pass time 12.76 +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 454 : backward pass time 10.14 +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 454 : optimizer step time 1.59 +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 454-------------------- +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:43:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.608896 41.700417 41.569344 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.687988 107.701172 107.648438 +util 65.000000 72.000000 61.000000 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 455 : avg iteration time 157.63 seconds +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 455 : data loading time 0.00 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 455 : avg forward pass time 70.24 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 455 : backward pass time 8.41 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 455 : optimizer step time 0.39 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 455-------------------- +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5654 iter count: 1.0 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5654 iter count: 1.0 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5654 iter count: 1.0 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5654iter count: 1.0 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:45:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.609560 41.701149 41.570526 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.363281 67.316406 +util 92.750000 99.000000 89.000000 +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 456 : avg iteration time 46.96 seconds +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 456 : data loading time 0.00 +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 456 : avg forward pass time 12.53 +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 456 : backward pass time 10.15 +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 456 : optimizer step time 1.58 +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 456-------------------- +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:46:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.609736 41.701157 41.570869 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.308594 67.316406 67.285156 +util 90.750000 98.000000 87.000000 +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 457 : avg iteration time 48.57 seconds +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 457 : data loading time 0.00 +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 457 : avg forward pass time 13.35 +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 457 : backward pass time 10.14 +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 457 : optimizer step time 1.56 +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 457-------------------- +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:47:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.610395 41.702496 41.571033 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.210938 67.304688 67.179688 +util 96.500000 99.000000 91.000000 +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 458 : avg iteration time 48.52 seconds +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 458 : data loading time 0.00 +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 458 : avg forward pass time 13.32 +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 458 : backward pass time 10.14 +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 458 : optimizer step time 1.57 +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 458-------------------- +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:48:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.610592 41.702950 41.571064 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.225586 67.304688 67.199219 +util 96.250000 100.000000 93.000000 +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 459 : avg iteration time 48.56 seconds +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 459 : data loading time 0.00 +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 459 : avg forward pass time 13.34 +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 459 : backward pass time 10.14 +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 459 : optimizer step time 1.58 +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 459-------------------- +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:48:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.611046 41.703304 41.571327 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.642578 107.628906 +util 64.000000 70.000000 60.000000 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 460 : avg iteration time 160.18 seconds +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 460 : data loading time 0.00 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 460 : avg forward pass time 71.50 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 460 : backward pass time 8.39 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 460 : optimizer step time 0.40 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 460-------------------- +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5566 iter count: 1.0 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5566 iter count: 1.0 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5566 iter count: 1.0 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5566iter count: 1.0 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:51:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.611655 41.703281 41.571617 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.071777 68.099609 67.988281 +util 98.250000 100.000000 95.000000 +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 461 : avg iteration time 44.31 seconds +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 461 : data loading time 0.00 +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 461 : avg forward pass time 11.23 +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 461 : backward pass time 10.14 +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 461 : optimizer step time 1.58 +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 461-------------------- +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:52:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.612059 41.703568 41.571648 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.071777 68.099609 67.988281 +util 98.000000 99.000000 95.000000 +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 462 : avg iteration time 43.40 seconds +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 462 : data loading time 0.00 +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 462 : avg forward pass time 10.77 +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 462 : backward pass time 10.14 +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 462 : optimizer step time 1.57 +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 462-------------------- +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:53:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.612414 41.703674 41.572742 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.920410 68.099609 67.382812 +util 96.750000 99.000000 92.000000 +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 463 : avg iteration time 45.95 seconds +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 463 : data loading time 0.00 +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 463 : avg forward pass time 12.04 +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 463 : backward pass time 10.14 +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 463 : optimizer step time 1.58 +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 463-------------------- +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:53:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.612777 41.704052 41.572941 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.149902 68.300781 68.099609 +util 93.250000 100.000000 89.000000 +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 464 : avg iteration time 45.90 seconds +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 464 : data loading time 0.00 +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 464 : avg forward pass time 12.01 +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 464 : backward pass time 10.15 +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 464 : optimizer step time 1.57 +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 464-------------------- +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:54:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.613000 41.703674 41.573097 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.668457 107.687500 107.662109 +util 61.000000 67.000000 55.000000 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 465 : avg iteration time 160.16 seconds +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 465 : data loading time 0.00 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 465 : avg forward pass time 71.51 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 465 : backward pass time 8.39 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 465 : optimizer step time 0.39 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 465-------------------- +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5283 iter count: 1.0 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5283 iter count: 1.0 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5283 iter count: 1.0 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5283iter count: 1.0 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:57:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.613171 41.703690 41.573223 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.777344 67.882812 67.460938 +util 96.000000 100.000000 93.000000 +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 466 : avg iteration time 44.27 seconds +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 466 : data loading time 0.00 +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 466 : avg forward pass time 11.21 +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 466 : backward pass time 10.16 +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 466 : optimizer step time 1.56 +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 466-------------------- +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:58:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.613269 41.703667 41.573467 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.718750 67.824219 67.402344 +util 98.000000 100.000000 94.000000 +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 467 : avg iteration time 48.47 seconds +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 467 : data loading time 0.00 +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 467 : avg forward pass time 13.31 +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 467 : backward pass time 10.15 +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 467 : optimizer step time 1.57 +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 467-------------------- +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:58:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.613502 41.703964 41.573578 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.752930 67.863281 67.421875 +util 95.250000 100.000000 92.000000 +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 468 : avg iteration time 48.51 seconds +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 468 : data loading time 0.00 +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 468 : avg forward pass time 13.32 +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 468 : backward pass time 10.14 +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 468 : optimizer step time 1.58 +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 468-------------------- +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 22:59:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.613996 41.704800 41.574478 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.382812 67.355469 +util 98.000000 100.000000 94.000000 +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 469 : avg iteration time 45.89 seconds +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 469 : data loading time 0.00 +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 469 : avg forward pass time 12.00 +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 469 : backward pass time 10.16 +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 469 : optimizer step time 1.57 +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 469-------------------- +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:00:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614003 41.703823 41.574478 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.687500 107.642578 +util 70.000000 73.000000 65.000000 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 470 : avg iteration time 154.82 seconds +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 470 : data loading time 0.00 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 470 : avg forward pass time 68.83 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 470 : backward pass time 8.40 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 470 : optimizer step time 0.39 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 470-------------------- +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5693 iter count: 1.0 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5693 iter count: 1.0 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5693 iter count: 1.0 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5693iter count: 1.0 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:02:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614306 41.703812 41.575470 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.290527 67.318359 67.207031 +util 91.500000 98.000000 88.000000 +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 471 : avg iteration time 44.23 seconds +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 471 : data loading time 0.00 +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 471 : avg forward pass time 11.19 +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 471 : backward pass time 10.13 +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 471 : optimizer step time 1.58 +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 471-------------------- +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:03:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614465 41.704144 41.575558 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.363770 67.441406 67.337891 +util 95.500000 98.000000 90.000000 +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 472 : avg iteration time 48.49 seconds +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 472 : data loading time 0.00 +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 472 : avg forward pass time 13.30 +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 472 : backward pass time 10.16 +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 472 : optimizer step time 1.57 +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 472-------------------- +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:04:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614557 41.704319 41.575573 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.568848 67.792969 67.494141 +util 98.250000 100.000000 95.000000 +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 473 : avg iteration time 45.86 seconds +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 473 : data loading time 0.00 +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 473 : avg forward pass time 12.00 +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 473 : backward pass time 10.13 +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 473 : optimizer step time 1.58 +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 473-------------------- +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:05:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614683 41.704315 41.575771 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.222168 67.285156 67.201172 +util 91.000000 98.000000 87.000000 +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 474 : avg iteration time 48.63 seconds +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 474 : data loading time 0.00 +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 474 : avg forward pass time 13.38 +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 474 : backward pass time 10.14 +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 474 : optimizer step time 1.57 +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 474-------------------- +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:06:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614670 41.704117 41.575809 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.521973 107.628906 107.486328 +util 65.250000 72.000000 62.000000 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 475 : avg iteration time 154.87 seconds +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 475 : data loading time 0.00 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 475 : avg forward pass time 68.84 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 475 : backward pass time 8.41 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 475 : optimizer step time 0.40 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 475-------------------- +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5571 iter count: 1.0 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5571 iter count: 1.0 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5571 iter count: 1.0 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5571iter count: 1.0 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:08:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614753 41.704250 41.575787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.168457 67.421875 67.083984 +util 92.000000 99.000000 88.000000 +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 476 : avg iteration time 46.94 seconds +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 476 : data loading time 0.00 +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 476 : avg forward pass time 12.54 +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 476 : backward pass time 10.15 +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 476 : optimizer step time 1.57 +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 476-------------------- +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:09:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614863 41.704491 41.575787 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.495605 67.533203 67.382812 +util 96.500000 99.000000 91.000000 +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 477 : avg iteration time 45.90 seconds +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 477 : data loading time 0.00 +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 477 : avg forward pass time 12.02 +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 477 : backward pass time 10.15 +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 477 : optimizer step time 1.56 +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 477-------------------- +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:10:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614963 41.704842 41.575802 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.329590 67.421875 67.298828 +util 93.750000 97.000000 88.000000 +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 478 : avg iteration time 45.97 seconds +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 478 : data loading time 0.00 +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 478 : avg forward pass time 12.05 +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 478 : backward pass time 10.14 +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 478 : optimizer step time 1.58 +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 478-------------------- +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:10:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615105 41.705090 41.575806 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.603027 67.669922 67.402344 +util 98.000000 99.000000 95.000000 +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 479 : avg iteration time 48.48 seconds +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 479 : data loading time 0.00 +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 479 : avg forward pass time 13.31 +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 479 : backward pass time 10.13 +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 479 : optimizer step time 1.57 +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 479-------------------- +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:11:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614779 41.703785 41.575817 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.731934 107.765625 107.720703 +util 68.000000 71.000000 62.000000 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 480 : avg iteration time 160.18 seconds +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 480 : data loading time 0.00 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 480 : avg forward pass time 71.52 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 480 : backward pass time 8.39 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 480 : optimizer step time 0.39 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 480-------------------- +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5269 iter count: 1.0 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5269 iter count: 1.0 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5269 iter count: 1.0 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5269iter count: 1.0 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:14:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614814 41.703785 41.575935 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.339355 67.343750 67.337891 +util 92.750000 99.000000 89.000000 +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 481 : avg iteration time 46.87 seconds +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 481 : data loading time 0.00 +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 481 : avg forward pass time 12.51 +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 481 : backward pass time 10.14 +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 481 : optimizer step time 1.58 +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 481-------------------- +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:15:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614926 41.704109 41.575928 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.319824 67.382812 67.298828 +util 97.000000 100.000000 91.000000 +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 482 : avg iteration time 48.51 seconds +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 482 : data loading time 0.00 +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 482 : avg forward pass time 13.32 +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 482 : backward pass time 10.14 +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 482 : optimizer step time 1.57 +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 482-------------------- +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:16:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614977 41.704109 41.576069 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.314941 67.318359 67.304688 +util 92.500000 99.000000 89.000000 +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 483 : avg iteration time 48.49 seconds +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 483 : data loading time 0.00 +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 483 : avg forward pass time 13.32 +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 483 : backward pass time 10.14 +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 483 : optimizer step time 1.56 +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 483-------------------- +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:16:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615015 41.704113 41.576256 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.148926 67.460938 67.044922 +util 96.500000 99.000000 91.000000 +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 484 : avg iteration time 43.29 seconds +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 484 : data loading time 0.00 +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 484 : avg forward pass time 10.71 +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 484 : backward pass time 10.14 +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 484 : optimizer step time 1.58 +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 484-------------------- +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:17:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.614994 41.703739 41.576241 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.770996 108.214844 107.623047 +util 68.250000 71.000000 63.000000 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 485 : avg iteration time 160.04 seconds +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 485 : data loading time 0.00 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 485 : avg forward pass time 71.45 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 485 : backward pass time 8.41 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 485 : optimizer step time 0.40 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 485-------------------- +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6035 iter count: 1.0 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6035 iter count: 1.0 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6035 iter count: 1.0 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6035iter count: 1.0 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:20:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615033 41.703735 41.576309 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.324707 67.402344 67.298828 +util 91.750000 98.000000 88.000000 +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 486 : avg iteration time 44.25 seconds +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 486 : data loading time 0.00 +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 486 : avg forward pass time 11.20 +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 486 : backward pass time 10.13 +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 486 : optimizer step time 1.56 +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 486-------------------- +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:20:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615103 41.703983 41.576321 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.358887 67.480469 67.318359 +util 96.500000 99.000000 91.000000 +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 487 : avg iteration time 48.51 seconds +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 487 : data loading time 0.00 +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 487 : avg forward pass time 13.32 +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 487 : backward pass time 10.16 +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 487 : optimizer step time 1.58 +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 487-------------------- +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:21:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615436 41.705223 41.576324 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.363770 67.500000 67.318359 +util 92.250000 99.000000 88.000000 +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 488 : avg iteration time 45.89 seconds +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 488 : data loading time 0.00 +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 488 : avg forward pass time 12.01 +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 488 : backward pass time 10.15 +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 488 : optimizer step time 1.56 +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 488-------------------- +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:22:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615461 41.705269 41.576359 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.354004 67.460938 67.318359 +util 96.250000 99.000000 90.000000 +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 489 : avg iteration time 45.90 seconds +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 489 : data loading time 0.00 +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 489 : avg forward pass time 12.01 +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 489 : backward pass time 10.15 +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 489 : optimizer step time 1.57 +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 489-------------------- +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:23:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615139 41.703918 41.576363 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.614746 107.648438 107.603516 +util 63.500000 70.000000 59.000000 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 490 : avg iteration time 162.73 seconds +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 490 : data loading time 0.00 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 490 : avg forward pass time 72.80 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 490 : backward pass time 8.38 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 490 : optimizer step time 0.39 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 490-------------------- +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5830 iter count: 1.0 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5830 iter count: 1.0 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5830 iter count: 1.0 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5830iter count: 1.0 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:26:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615225 41.703953 41.576385 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.372070 67.375000 67.363281 +util 97.750000 100.000000 93.000000 +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 491 : avg iteration time 47.10 seconds +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 491 : data loading time 0.00 +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 491 : avg forward pass time 12.63 +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 491 : backward pass time 10.12 +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 491 : optimizer step time 1.57 +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 491-------------------- +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:26:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615256 41.703972 41.576363 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.337891 67.375000 67.226562 +util 93.000000 99.000000 88.000000 +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 492 : avg iteration time 45.84 seconds +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 492 : data loading time 0.00 +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 492 : avg forward pass time 11.98 +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 492 : backward pass time 10.15 +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 492 : optimizer step time 1.56 +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 492-------------------- +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:27:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615522 41.704948 41.576370 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.391602 67.441406 67.375000 +util 96.750000 99.000000 91.000000 +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 493 : avg iteration time 45.94 seconds +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 493 : data loading time 0.00 +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 493 : avg forward pass time 12.03 +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 493 : backward pass time 10.14 +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 493 : optimizer step time 1.58 +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 493-------------------- +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:28:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615623 41.705185 41.576412 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.352539 67.375000 67.285156 +util 93.000000 99.000000 89.000000 +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 494 : avg iteration time 45.90 seconds +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 494 : data loading time 0.00 +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 494 : avg forward pass time 12.02 +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 494 : backward pass time 10.15 +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 494 : optimizer step time 1.57 +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 494-------------------- +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:29:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615773 41.705673 41.576393 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.642578 107.609375 +util 64.750000 68.000000 59.000000 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 495 : avg iteration time 160.09 seconds +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 495 : data loading time 0.00 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 495 : avg forward pass time 71.47 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 495 : backward pass time 8.40 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 495 : optimizer step time 0.39 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 495-------------------- +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6104 iter count: 1.0 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6104 iter count: 1.0 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6104 iter count: 1.0 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6104iter count: 1.0 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:31:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615586 41.704929 41.576397 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 66.997559 67.148438 66.947266 +util 96.500000 99.000000 91.000000 +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 496 : avg iteration time 47.01 seconds +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 496 : data loading time 0.00 +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 496 : avg forward pass time 12.55 +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 496 : backward pass time 10.16 +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 496 : optimizer step time 1.59 +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 496-------------------- +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:32:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615593 41.704929 41.576401 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.026855 67.265625 66.947266 +util 95.000000 98.000000 89.000000 +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 497 : avg iteration time 43.38 seconds +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 497 : data loading time 0.00 +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 497 : avg forward pass time 10.74 +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 497 : backward pass time 10.15 +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 497 : optimizer step time 1.57 +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 497-------------------- +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:33:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615602 41.704948 41.576412 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 66.978027 67.070312 66.947266 +util 95.500000 100.000000 88.000000 +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 498 : avg iteration time 45.88 seconds +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 498 : data loading time 0.00 +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 498 : avg forward pass time 12.00 +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 498 : backward pass time 10.14 +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 498 : optimizer step time 1.59 +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 498-------------------- +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:34:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615713 41.705177 41.576500 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 66.978027 67.070312 66.947266 +util 91.750000 98.000000 88.000000 +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 499 : avg iteration time 45.83 seconds +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 499 : data loading time 0.00 +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 499 : avg forward pass time 11.99 +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 499 : backward pass time 10.14 +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 499 : optimizer step time 1.57 +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 499-------------------- +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:34:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.615803 41.705452 41.576500 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.624512 107.746094 107.583984 +util 64.000000 70.000000 60.000000 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 500 : avg iteration time 160.05 seconds +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 500 : data loading time 0.00 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 500 : avg forward pass time 71.44 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 500 : backward pass time 8.41 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 500 : optimizer step time 0.40 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 500-------------------- +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5635 iter count: 1.0 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5635 iter count: 1.0 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5635 iter count: 1.0 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5635iter count: 1.0 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 4, 2026 - 23:37:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 23:37:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.22 seconds +[May 4, 2026 - 23:37:43 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 500 +[May 4, 2026 - 23:37:43 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000500 +[May 4, 2026 - 23:38:01 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 500 +[May 4, 2026 - 23:38:01 | INFO | fastgen.trainer:validate:358 ] Validation iteration 500 +[May 4, 2026 - 23:38:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 4, 2026 - 23:38:06 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.125 +[May 4, 2026 - 23:38:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 23:38:12 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.139 +[May 4, 2026 - 23:38:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 23:38:17 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.746 +[May 4, 2026 - 23:38:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 23:38:22 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.866 +[May 4, 2026 - 23:38:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 23:38:27 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=5.030 +[May 4, 2026 - 23:38:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 4, 2026 - 23:38:32 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.340 +[May 4, 2026 - 23:38:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 23:38:37 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.626 +[May 4, 2026 - 23:38:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 23:38:42 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.420 +[May 4, 2026 - 23:38:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 23:38:47 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.517 +[May 4, 2026 - 23:38:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 4, 2026 - 23:38:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 4, 2026 - 23:38:51 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.112 +[May 4, 2026 - 23:38:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 500-------------------- +[May 4, 2026 - 23:38:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 4, 2026 - 23:39:08 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 500 +[May 4, 2026 - 23:39:08 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.292 (n=10) +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617250 41.710732 41.576717 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.397461 68.593750 68.332031 +util 94.000000 100.000000 89.000000 +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 501 : avg iteration time 147.33 seconds +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 0.00 +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 12.99 +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 10.14 +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 1.56 +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 501-------------------- +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:39:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617277 41.710686 41.576729 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.377930 68.574219 68.312500 +util 90.750000 97.000000 87.000000 +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 46.01 seconds +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 12.05 +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 10.16 +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 1.58 +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 502-------------------- +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:40:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617273 41.710705 41.576740 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.387695 68.613281 68.312500 +util 94.250000 99.000000 91.000000 +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 46.05 seconds +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 12.07 +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 10.17 +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 1.57 +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 503-------------------- +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:41:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617324 41.710812 41.576767 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.285156 68.613281 68.175781 +util 94.000000 100.000000 90.000000 +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 48.59 seconds +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 13.36 +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 10.14 +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 1.58 +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 504-------------------- +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:42:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616295 41.706753 41.576752 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.648438 107.583984 +util 68.250000 72.000000 62.000000 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 160.12 seconds +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 71.48 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 8.41 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.39 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 505-------------------- +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5229 iter count: 1.0 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5229 iter count: 1.0 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5229 iter count: 1.0 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5229iter count: 1.0 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:44:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616363 41.706764 41.576782 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.630859 67.726562 67.343750 +util 95.250000 100.000000 92.000000 +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 46.91 seconds +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 12.51 +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 10.14 +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 1.57 +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 506-------------------- +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:45:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616617 41.707691 41.576771 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.591797 67.765625 67.070312 +util 96.000000 99.000000 90.000000 +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 45.86 seconds +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 11.99 +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 10.16 +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 1.57 +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 507-------------------- +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:46:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616652 41.707787 41.576794 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.567383 67.726562 67.089844 +util 91.500000 98.000000 87.000000 +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 45.90 seconds +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 12.03 +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 10.13 +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 1.57 +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 508-------------------- +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:47:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616709 41.708031 41.576817 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.713867 67.765625 67.558594 +util 96.750000 100.000000 94.000000 +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 48.47 seconds +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 13.31 +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 10.14 +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 1.55 +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 509-------------------- +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:48:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616549 41.707268 41.576817 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.639160 107.687500 107.623047 +util 65.000000 67.000000 63.000000 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 157.62 seconds +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 70.23 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 8.40 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.39 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 510-------------------- +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5557 iter count: 1.0 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5557 iter count: 1.0 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5557 iter count: 1.0 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5557iter count: 1.0 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:50:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616558 41.707268 41.576817 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.694336 67.785156 67.421875 +util 94.500000 100.000000 91.000000 +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 49.55 seconds +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 13.86 +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 10.14 +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 1.56 +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 511-------------------- +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:51:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616596 41.707314 41.576843 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.045898 68.312500 67.246094 +util 97.250000 99.000000 92.000000 +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 45.90 seconds +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 12.02 +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 10.14 +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 1.57 +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 512-------------------- +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:52:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616869 41.708263 41.576824 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.016602 68.332031 67.070312 +util 96.500000 99.000000 94.000000 +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 43.33 seconds +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 10.74 +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 10.14 +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 1.58 +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 513-------------------- +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:53:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616879 41.708271 41.576824 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.026367 68.332031 67.109375 +util 92.500000 100.000000 87.000000 +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 43.30 seconds +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 10.70 +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 10.15 +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 1.58 +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 514-------------------- +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:53:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616867 41.708141 41.576836 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.589844 107.583984 +util 66.500000 74.000000 57.000000 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 160.20 seconds +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 71.52 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 8.41 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.39 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 515-------------------- +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5986 iter count: 1.0 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5986 iter count: 1.0 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5986 iter count: 1.0 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5986iter count: 1.0 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:56:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616936 41.708256 41.576973 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.744629 67.865234 67.382812 +util 96.250000 99.000000 90.000000 +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 49.54 seconds +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 13.83 +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 10.17 +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 1.57 +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 516-------------------- +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:57:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616957 41.708340 41.576996 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.759277 67.865234 67.441406 +util 97.500000 100.000000 92.000000 +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 43.22 seconds +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 10.68 +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 10.14 +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 1.58 +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 517-------------------- +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:57:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617057 41.708538 41.576996 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.671387 67.845703 67.148438 +util 97.750000 100.000000 93.000000 +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 43.21 seconds +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 10.69 +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 10.13 +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 1.56 +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 518-------------------- +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:58:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617066 41.708553 41.576984 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.686035 67.865234 67.148438 +util 90.500000 96.000000 88.000000 +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 45.86 seconds +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 11.99 +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 10.14 +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 1.58 +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 519-------------------- +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 4, 2026 - 23:59:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616811 41.707508 41.577011 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.600098 107.603516 107.589844 +util 66.500000 73.000000 62.000000 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 165.08 seconds +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 73.96 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 8.41 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.40 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 520-------------------- +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5332 iter count: 1.0 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5332 iter count: 1.0 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5332 iter count: 1.0 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5332iter count: 1.0 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:02:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616730 41.707130 41.577030 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.686035 67.845703 67.207031 +util 95.250000 100.000000 92.000000 +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 46.97 seconds +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 12.56 +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 10.14 +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 1.57 +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 521-------------------- +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:02:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616733 41.707161 41.577011 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.739746 67.826172 67.480469 +util 96.250000 99.000000 90.000000 +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 48.46 seconds +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.00 +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 13.30 +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 10.14 +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 1.56 +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 522-------------------- +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:03:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616796 41.707413 41.577015 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.734863 67.826172 67.460938 +util 98.000000 100.000000 93.000000 +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 43.24 seconds +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 10.70 +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 10.12 +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 1.57 +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 523-------------------- +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:04:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616817 41.707420 41.577045 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.666504 67.826172 67.187500 +util 93.250000 99.000000 89.000000 +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 524 : avg iteration time 48.47 seconds +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 524 : data loading time 0.00 +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 524 : avg forward pass time 13.31 +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 524 : backward pass time 10.15 +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 524 : optimizer step time 1.57 +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 524-------------------- +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:05:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616785 41.707253 41.577034 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.765625 107.505859 +util 63.000000 69.000000 59.000000 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 525 : avg iteration time 157.41 seconds +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 525 : data loading time 0.00 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 525 : avg forward pass time 70.14 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 525 : backward pass time 8.39 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 525 : optimizer step time 0.39 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 525-------------------- +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5269 iter count: 1.0 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5269 iter count: 1.0 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5269 iter count: 1.0 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5269iter count: 1.0 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:07:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616786 41.707188 41.577030 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.622559 67.787109 67.128906 +util 97.000000 100.000000 91.000000 +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 526 : avg iteration time 44.30 seconds +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 526 : data loading time 0.00 +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 526 : avg forward pass time 11.21 +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 526 : backward pass time 10.16 +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 526 : optimizer step time 1.58 +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 526-------------------- +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:08:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616809 41.707188 41.577034 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.110840 68.412109 67.207031 +util 91.500000 98.000000 87.000000 +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 527 : avg iteration time 48.48 seconds +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 527 : data loading time 0.00 +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 527 : avg forward pass time 13.30 +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 527 : backward pass time 10.14 +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 527 : optimizer step time 1.56 +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 527-------------------- +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:09:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616861 41.707188 41.577034 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.115723 68.412109 67.226562 +util 97.000000 100.000000 91.000000 +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 528 : avg iteration time 43.33 seconds +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 528 : data loading time 0.00 +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 528 : avg forward pass time 10.71 +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 528 : backward pass time 10.16 +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 528 : optimizer step time 1.58 +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 528-------------------- +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:10:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616921 41.707188 41.577042 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.110840 68.412109 67.207031 +util 93.500000 99.000000 90.000000 +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 529 : avg iteration time 48.69 seconds +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 529 : data loading time 0.00 +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 529 : avg forward pass time 13.40 +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 529 : backward pass time 10.15 +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 529 : optimizer step time 1.58 +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 529-------------------- +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:11:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616932 41.707184 41.577034 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.687988 107.707031 107.681641 +util 66.500000 71.000000 60.000000 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 530 : avg iteration time 157.30 seconds +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 530 : data loading time 0.00 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 530 : avg forward pass time 70.08 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 530 : backward pass time 8.38 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 530 : optimizer step time 0.39 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 530-------------------- +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5996 iter count: 1.0 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5996 iter count: 1.0 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5996 iter count: 1.0 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5996iter count: 1.0 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:13:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616987 41.707169 41.577042 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.407715 67.500000 67.376953 +util 96.750000 99.000000 91.000000 +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 531 : avg iteration time 49.60 seconds +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 531 : data loading time 0.00 +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 531 : avg forward pass time 13.86 +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 531 : backward pass time 10.15 +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 531 : optimizer step time 1.58 +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 531-------------------- +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:14:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617001 41.707169 41.577042 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.300293 67.539062 67.220703 +util 95.750000 100.000000 92.000000 +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 532 : avg iteration time 48.53 seconds +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 532 : data loading time 0.00 +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 532 : avg forward pass time 13.32 +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 532 : backward pass time 10.15 +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 532 : optimizer step time 1.57 +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 532-------------------- +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:15:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617018 41.707260 41.577042 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.388184 67.396484 67.363281 +util 96.500000 100.000000 90.000000 +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 533 : avg iteration time 45.86 seconds +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 533 : data loading time 0.00 +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 533 : avg forward pass time 12.01 +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 533 : backward pass time 10.14 +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 533 : optimizer step time 1.56 +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 533-------------------- +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:16:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617010 41.707287 41.577042 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.402832 67.416016 67.363281 +util 92.250000 98.000000 88.000000 +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 534 : avg iteration time 45.89 seconds +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 534 : data loading time 0.00 +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 534 : avg forward pass time 12.01 +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 534 : backward pass time 10.16 +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 534 : optimizer step time 1.58 +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 534-------------------- +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:16:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616862 41.706642 41.577072 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.619629 107.623047 107.609375 +util 65.750000 72.000000 62.000000 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 535 : avg iteration time 157.46 seconds +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 535 : data loading time 0.00 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 535 : avg forward pass time 70.15 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 535 : backward pass time 8.40 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 535 : optimizer step time 0.39 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 535-------------------- +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5566 iter count: 1.0 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5566 iter count: 1.0 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5566 iter count: 1.0 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5566iter count: 1.0 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:19:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616875 41.706665 41.577072 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.308594 67.343750 67.296875 +util 97.500000 100.000000 92.000000 +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 536 : avg iteration time 46.88 seconds +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 536 : data loading time 0.00 +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 536 : avg forward pass time 12.52 +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 536 : backward pass time 10.13 +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 536 : optimizer step time 1.57 +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 536-------------------- +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:20:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617074 41.707417 41.577095 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.274414 67.335938 67.089844 +util 94.000000 99.000000 90.000000 +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 537 : avg iteration time 48.49 seconds +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 537 : data loading time 0.00 +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 537 : avg forward pass time 13.32 +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 537 : backward pass time 10.13 +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 537 : optimizer step time 1.58 +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 537-------------------- +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:21:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617070 41.707405 41.577103 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.391602 67.500000 67.355469 +util 94.000000 100.000000 90.000000 +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 538 : avg iteration time 45.80 seconds +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 538 : data loading time 0.00 +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 538 : avg forward pass time 11.98 +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 538 : backward pass time 10.13 +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 538 : optimizer step time 1.57 +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 538-------------------- +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:21:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617131 41.707642 41.577103 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.328125 67.433594 67.011719 +util 95.750000 99.000000 89.000000 +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 539 : avg iteration time 48.47 seconds +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 539 : data loading time 0.00 +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 539 : avg forward pass time 13.30 +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 539 : backward pass time 10.16 +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 539 : optimizer step time 1.57 +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 539-------------------- +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:22:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617031 41.707172 41.577129 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.575684 107.726562 107.525391 +util 65.750000 70.000000 59.000000 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 540 : avg iteration time 160.02 seconds +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 540 : data loading time 0.00 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 540 : avg forward pass time 71.44 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 540 : backward pass time 8.39 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 540 : optimizer step time 0.40 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 540-------------------- +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5889 iter count: 1.0 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5889 iter count: 1.0 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5889 iter count: 1.0 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5889iter count: 1.0 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:25:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617027 41.707172 41.577110 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.128906 68.371094 67.402344 +util 93.250000 99.000000 89.000000 +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 541 : avg iteration time 49.50 seconds +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 541 : data loading time 0.00 +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 541 : avg forward pass time 13.82 +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 541 : backward pass time 10.14 +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 541 : optimizer step time 1.57 +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 541-------------------- +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:26:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617097 41.707420 41.577118 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.125488 68.373047 67.382812 +util 91.250000 97.000000 87.000000 +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 542 : avg iteration time 45.98 seconds +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 542 : data loading time 0.00 +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 542 : avg forward pass time 12.06 +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 542 : backward pass time 10.14 +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 542 : optimizer step time 1.56 +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 542-------------------- +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:26:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617144 41.707420 41.577133 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.145020 68.392578 67.402344 +util 96.750000 99.000000 91.000000 +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 543 : avg iteration time 45.93 seconds +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 543 : data loading time 0.00 +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 543 : avg forward pass time 12.04 +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 543 : backward pass time 10.13 +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 543 : optimizer step time 1.57 +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 543-------------------- +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:27:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617214 41.707661 41.577133 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.086426 68.373047 67.226562 +util 93.750000 100.000000 89.000000 +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 544 : avg iteration time 48.46 seconds +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 544 : data loading time 0.00 +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 544 : avg forward pass time 13.31 +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 544 : backward pass time 10.14 +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 544 : optimizer step time 1.57 +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 544-------------------- +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:28:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617010 41.706715 41.577168 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.570801 107.648438 107.544922 +util 65.500000 71.000000 61.000000 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 545 : avg iteration time 160.08 seconds +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 545 : data loading time 0.00 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 545 : avg forward pass time 71.46 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 545 : backward pass time 8.39 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 545 : optimizer step time 0.39 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 545-------------------- +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.6309 iter count: 1.0 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6309 iter count: 1.0 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.6309 iter count: 1.0 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6309iter count: 1.0 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:31:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616993 41.706707 41.577122 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.323242 67.402344 67.296875 +util 93.250000 99.000000 89.000000 +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 546 : avg iteration time 46.95 seconds +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 546 : data loading time 0.00 +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 546 : avg forward pass time 12.55 +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 546 : backward pass time 10.14 +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 546 : optimizer step time 1.57 +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 546-------------------- +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:31:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617077 41.706955 41.577164 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.352539 67.460938 67.316406 +util 92.250000 98.000000 88.000000 +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 547 : avg iteration time 43.19 seconds +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 547 : data loading time 0.00 +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 547 : avg forward pass time 10.68 +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 547 : backward pass time 10.13 +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 547 : optimizer step time 1.57 +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 547-------------------- +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:32:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617086 41.706955 41.577141 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.382812 67.355469 +util 91.250000 97.000000 87.000000 +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 548 : avg iteration time 48.58 seconds +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 548 : data loading time 0.00 +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 548 : avg forward pass time 13.38 +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 548 : backward pass time 10.13 +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 548 : optimizer step time 1.55 +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 548-------------------- +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:33:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617111 41.706985 41.577164 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.362305 67.441406 67.335938 +util 97.000000 100.000000 91.000000 +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 549 : avg iteration time 48.52 seconds +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 549 : data loading time 0.00 +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 549 : avg forward pass time 13.32 +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 549 : backward pass time 10.14 +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 549 : optimizer step time 1.57 +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 549-------------------- +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:34:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616928 41.706276 41.577141 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.629395 107.707031 107.603516 +util 61.500000 67.000000 57.000000 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 550 : avg iteration time 154.90 seconds +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 550 : data loading time 0.00 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 550 : avg forward pass time 68.89 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 550 : backward pass time 8.39 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 550 : optimizer step time 0.39 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 550-------------------- +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5459 iter count: 1.0 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5459 iter count: 1.0 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5459 iter count: 1.0 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5459iter count: 1.0 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:36:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.616932 41.706276 41.577152 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.645508 67.804688 67.167969 +util 91.250000 97.000000 87.000000 +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 551 : avg iteration time 44.38 seconds +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 551 : data loading time 0.00 +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 551 : avg forward pass time 11.26 +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 551 : backward pass time 10.15 +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 551 : optimizer step time 1.57 +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 551-------------------- +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:37:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617004 41.706478 41.577141 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.001953 68.292969 67.128906 +util 97.000000 99.000000 94.000000 +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 552 : avg iteration time 45.89 seconds +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 552 : data loading time 0.00 +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 552 : avg forward pass time 12.00 +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 552 : backward pass time 10.16 +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 552 : optimizer step time 1.57 +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 552-------------------- +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:38:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617316 41.707741 41.577141 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.021484 68.312500 67.148438 +util 97.750000 100.000000 92.000000 +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 553 : avg iteration time 45.86 seconds +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 553 : data loading time 0.00 +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 553 : avg forward pass time 12.00 +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 553 : backward pass time 10.13 +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 553 : optimizer step time 1.56 +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 553-------------------- +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:39:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617412 41.708000 41.577156 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.011719 68.312500 67.109375 +util 90.750000 96.000000 87.000000 +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 554 : avg iteration time 45.86 seconds +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 554 : data loading time 0.00 +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 554 : avg forward pass time 12.02 +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 554 : backward pass time 10.12 +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 554 : optimizer step time 1.56 +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 554-------------------- +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:39:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617015 41.706280 41.577152 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.595215 107.628906 107.583984 +util 65.750000 71.000000 62.000000 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 555 : avg iteration time 160.27 seconds +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 555 : data loading time 0.00 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 555 : avg forward pass time 71.58 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 555 : backward pass time 8.37 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 555 : optimizer step time 0.39 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 555-------------------- +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5391 iter count: 1.0 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5391 iter count: 1.0 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5391 iter count: 1.0 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5391iter count: 1.0 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:42:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617022 41.706318 41.577164 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.674805 67.765625 67.402344 +util 94.750000 99.000000 87.000000 +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 556 : avg iteration time 46.82 seconds +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 556 : data loading time 0.00 +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 556 : avg forward pass time 12.50 +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 556 : backward pass time 10.14 +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 556 : optimizer step time 1.56 +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 556-------------------- +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0164 iter count: 1.0 +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:43:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617079 41.706524 41.577164 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.665039 67.765625 67.363281 +util 95.250000 100.000000 89.000000 +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 557 : avg iteration time 48.47 seconds +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 557 : data loading time 0.00 +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 557 : avg forward pass time 13.31 +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 557 : backward pass time 10.15 +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 557 : optimizer step time 1.56 +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 557-------------------- +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:44:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617333 41.707531 41.577175 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.669922 67.765625 67.382812 +util 97.000000 99.000000 91.000000 +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 558 : avg iteration time 43.18 seconds +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 558 : data loading time 0.00 +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 558 : avg forward pass time 10.68 +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 558 : backward pass time 10.12 +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 558 : optimizer step time 1.56 +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 558-------------------- +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:44:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617351 41.707615 41.577175 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.669922 67.765625 67.382812 +util 91.000000 96.000000 87.000000 +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 559 : avg iteration time 45.86 seconds +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 559 : data loading time 0.00 +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 559 : avg forward pass time 12.00 +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 559 : backward pass time 10.14 +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 559 : optimizer step time 1.57 +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 559-------------------- +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:45:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617453 41.707855 41.577175 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.589844 107.583984 +util 62.250000 68.000000 58.000000 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 560 : avg iteration time 157.43 seconds +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 560 : data loading time 0.00 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 560 : avg forward pass time 70.15 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 560 : backward pass time 8.39 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 560 : optimizer step time 0.39 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 560-------------------- +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5386 iter count: 1.0 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5386 iter count: 1.0 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5386 iter count: 1.0 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5386iter count: 1.0 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:48:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617283 41.707233 41.577175 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.720215 67.845703 67.343750 +util 96.000000 99.000000 89.000000 +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 561 : avg iteration time 46.88 seconds +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 561 : data loading time 0.00 +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 561 : avg forward pass time 12.52 +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 561 : backward pass time 10.14 +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 561 : optimizer step time 1.57 +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 561-------------------- +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:48:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617297 41.707237 41.577187 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.018066 68.314453 67.128906 +util 94.500000 99.000000 87.000000 +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 562 : avg iteration time 48.44 seconds +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 562 : data loading time 0.00 +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 562 : avg forward pass time 13.29 +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 562 : backward pass time 10.14 +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 562 : optimizer step time 1.55 +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 562-------------------- +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:49:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617382 41.707558 41.577187 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.105957 68.353516 67.363281 +util 94.750000 100.000000 87.000000 +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 563 : avg iteration time 43.21 seconds +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 563 : data loading time 0.00 +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 563 : avg forward pass time 10.68 +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 563 : backward pass time 10.14 +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 563 : optimizer step time 1.57 +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 563-------------------- +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:50:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617382 41.707527 41.577187 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.076660 68.392578 67.128906 +util 94.750000 100.000000 91.000000 +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 564 : avg iteration time 45.88 seconds +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 564 : data loading time 0.00 +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 564 : avg forward pass time 12.02 +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 564 : backward pass time 10.15 +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 564 : optimizer step time 1.56 +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 564-------------------- +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:51:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617439 41.707756 41.577190 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 108.225098 108.423828 107.628906 +util 59.750000 66.000000 55.000000 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 565 : avg iteration time 157.55 seconds +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 565 : data loading time 0.00 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 565 : avg forward pass time 70.21 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 565 : backward pass time 8.39 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 565 : optimizer step time 0.39 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 565-------------------- +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5137 iter count: 1.0 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5137 iter count: 1.0 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5137 iter count: 1.0 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5137iter count: 1.0 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:53:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617460 41.707779 41.577194 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.852051 68.001953 67.402344 +util 95.750000 99.000000 89.000000 +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 566 : avg iteration time 46.91 seconds +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 566 : data loading time 0.00 +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 566 : avg forward pass time 12.54 +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 566 : backward pass time 10.13 +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 566 : optimizer step time 1.56 +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 566-------------------- +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:54:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617473 41.707771 41.577202 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.798340 68.001953 67.187500 +util 97.000000 99.000000 91.000000 +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 567 : avg iteration time 43.23 seconds +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 567 : data loading time 0.00 +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 567 : avg forward pass time 10.67 +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 567 : backward pass time 10.15 +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 567 : optimizer step time 1.58 +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 567-------------------- +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:55:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617473 41.707790 41.577229 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.803223 68.001953 67.207031 +util 97.250000 99.000000 92.000000 +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 568 : avg iteration time 45.82 seconds +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 568 : data loading time 0.00 +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 568 : avg forward pass time 11.98 +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 568 : backward pass time 10.14 +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 568 : optimizer step time 1.57 +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 568-------------------- +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:56:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617596 41.708275 41.577213 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.783691 68.001953 67.128906 +util 95.250000 99.000000 92.000000 +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 569 : avg iteration time 45.91 seconds +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 569 : data loading time 0.00 +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 569 : avg forward pass time 12.04 +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 569 : backward pass time 10.15 +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 569 : optimizer step time 1.55 +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 569-------------------- +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:56:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617359 41.707359 41.577225 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.687500 107.642578 +util 61.750000 67.000000 57.000000 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 570 : avg iteration time 160.37 seconds +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 570 : data loading time 0.00 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 570 : avg forward pass time 71.61 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 570 : backward pass time 8.40 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 570 : optimizer step time 0.40 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 570-------------------- +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5928 iter count: 1.0 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5928 iter count: 1.0 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5928 iter count: 1.0 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5928iter count: 1.0 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 00:59:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617355 41.707352 41.577217 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.646973 67.826172 67.109375 +util 91.250000 97.000000 87.000000 +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 571 : avg iteration time 49.57 seconds +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 571 : data loading time 0.00 +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 571 : avg forward pass time 13.86 +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 571 : backward pass time 10.15 +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 571 : optimizer step time 1.57 +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 571-------------------- +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:00:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617373 41.707375 41.577251 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.695801 67.884766 67.128906 +util 95.750000 100.000000 92.000000 +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 572 : avg iteration time 43.26 seconds +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 572 : data loading time 0.00 +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 572 : avg forward pass time 10.70 +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 572 : backward pass time 10.15 +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 572 : optimizer step time 1.56 +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 572-------------------- +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:01:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617374 41.707382 41.577229 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.666504 67.845703 67.128906 +util 95.750000 100.000000 92.000000 +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 573 : avg iteration time 46.20 seconds +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 573 : data loading time 0.00 +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 573 : avg forward pass time 12.16 +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 573 : backward pass time 10.15 +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 573 : optimizer step time 1.57 +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 573-------------------- +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:01:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617377 41.707352 41.577259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.690918 67.845703 67.226562 +util 94.000000 99.000000 90.000000 +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 574 : avg iteration time 43.58 seconds +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 574 : data loading time 0.00 +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 574 : avg forward pass time 10.87 +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 574 : backward pass time 10.14 +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 574 : optimizer step time 1.57 +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 574-------------------- +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:02:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617290 41.706924 41.577259 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.556152 107.648438 107.525391 +util 67.250000 72.000000 60.000000 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 575 : avg iteration time 163.70 seconds +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 575 : data loading time 0.00 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 575 : avg forward pass time 73.22 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 575 : backward pass time 8.40 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 575 : optimizer step time 0.39 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 575-------------------- +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5527 iter count: 1.0 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5527 iter count: 1.0 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5527 iter count: 1.0 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5527iter count: 1.0 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:05:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617294 41.706902 41.577259 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.611328 67.804688 67.031250 +util 92.000000 97.000000 88.000000 +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 576 : avg iteration time 50.27 seconds +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 576 : data loading time 0.00 +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 576 : avg forward pass time 14.20 +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 576 : backward pass time 10.15 +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 576 : optimizer step time 1.57 +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 576-------------------- +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:06:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617358 41.707161 41.577293 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.207031 68.488281 67.363281 +util 95.000000 100.000000 88.000000 +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 577 : avg iteration time 43.56 seconds +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 577 : data loading time 0.00 +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 577 : avg forward pass time 10.85 +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 577 : backward pass time 10.13 +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 577 : optimizer step time 1.58 +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 577-------------------- +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:06:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617358 41.707150 41.577293 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.080078 68.312500 67.382812 +util 95.250000 99.000000 89.000000 +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 578 : avg iteration time 46.12 seconds +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 578 : data loading time 0.00 +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 578 : avg forward pass time 12.12 +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 578 : backward pass time 10.16 +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 578 : optimizer step time 1.57 +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 578-------------------- +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:07:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617361 41.707165 41.577297 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.997070 68.292969 67.109375 +util 95.500000 100.000000 92.000000 +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 579 : avg iteration time 46.16 seconds +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 579 : data loading time 0.00 +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 579 : avg forward pass time 12.14 +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 579 : backward pass time 10.16 +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 579 : optimizer step time 1.58 +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 579-------------------- +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:08:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617313 41.706944 41.577305 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.561035 107.667969 107.525391 +util 65.500000 70.000000 58.000000 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 580 : avg iteration time 161.48 seconds +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 580 : data loading time 0.00 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 580 : avg forward pass time 72.14 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 580 : backward pass time 8.40 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 580 : optimizer step time 0.40 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 580-------------------- +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5430 iter count: 1.0 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5430 iter count: 1.0 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5430 iter count: 1.0 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5430iter count: 1.0 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:11:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617320 41.706955 41.577293 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.411133 67.578125 67.355469 +util 97.250000 100.000000 91.000000 +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 581 : avg iteration time 47.21 seconds +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 581 : data loading time 0.00 +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 581 : avg forward pass time 12.68 +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 581 : backward pass time 10.18 +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 581 : optimizer step time 1.56 +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 581-------------------- +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:11:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617646 41.708233 41.577305 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.674805 67.843750 67.167969 +util 97.500000 100.000000 91.000000 +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 582 : avg iteration time 43.39 seconds +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 582 : data loading time 0.00 +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 582 : avg forward pass time 10.78 +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 582 : backward pass time 10.13 +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 582 : optimizer step time 1.57 +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 582-------------------- +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:12:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617679 41.708248 41.577282 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.669922 67.843750 67.148438 +util 95.000000 100.000000 88.000000 +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 583 : avg iteration time 46.02 seconds +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 583 : data loading time 0.00 +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 583 : avg forward pass time 12.08 +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 583 : backward pass time 10.13 +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 583 : optimizer step time 1.58 +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 583-------------------- +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:13:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617666 41.708229 41.577282 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.669922 67.843750 67.148438 +util 94.750000 99.000000 91.000000 +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 584 : avg iteration time 46.09 seconds +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 584 : data loading time 0.00 +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 584 : avg forward pass time 12.11 +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 584 : backward pass time 10.16 +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 584 : optimizer step time 1.56 +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 584-------------------- +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:14:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617398 41.707115 41.577293 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.634277 107.726562 107.603516 +util 65.500000 70.000000 58.000000 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 585 : avg iteration time 163.67 seconds +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 585 : data loading time 0.00 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 585 : avg forward pass time 73.23 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 585 : backward pass time 8.39 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 585 : optimizer step time 0.40 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 585-------------------- +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5879 iter count: 1.0 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5879 iter count: 1.0 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5879 iter count: 1.0 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5879iter count: 1.0 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:16:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617393 41.707100 41.577286 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.279297 67.335938 67.109375 +util 94.000000 99.000000 90.000000 +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 586 : avg iteration time 44.51 seconds +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 586 : data loading time 0.00 +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 586 : avg forward pass time 11.32 +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 586 : backward pass time 10.13 +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 586 : optimizer step time 1.57 +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 586-------------------- +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:17:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617402 41.707111 41.577309 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.357422 67.421875 67.335938 +util 97.000000 99.000000 94.000000 +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 587 : avg iteration time 43.45 seconds +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 587 : data loading time 0.00 +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 587 : avg forward pass time 10.79 +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 587 : backward pass time 10.15 +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 587 : optimizer step time 1.57 +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 587-------------------- +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:18:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617709 41.708363 41.577286 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.298828 67.355469 67.128906 +util 94.250000 99.000000 87.000000 +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 588 : avg iteration time 45.87 seconds +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 588 : data loading time 0.00 +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 588 : avg forward pass time 12.02 +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 588 : backward pass time 10.12 +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 588 : optimizer step time 1.57 +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 588-------------------- +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:19:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617723 41.708351 41.577305 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.474609 67.492188 67.421875 +util 95.750000 100.000000 92.000000 +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 589 : avg iteration time 46.11 seconds +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 589 : data loading time 0.00 +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 589 : avg forward pass time 12.14 +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 589 : backward pass time 10.12 +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 589 : optimizer step time 1.56 +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 589-------------------- +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:19:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617266 41.706524 41.577286 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.653809 107.746094 107.623047 +util 68.500000 74.000000 64.000000 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 590 : avg iteration time 158.05 seconds +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 590 : data loading time 0.00 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 590 : avg forward pass time 70.45 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 590 : backward pass time 8.38 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 590 : optimizer step time 0.40 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 590-------------------- +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5449 iter count: 1.0 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5449 iter count: 1.0 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5449 iter count: 1.0 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5449iter count: 1.0 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:22:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617263 41.706509 41.577286 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.344238 67.421875 67.318359 +util 93.000000 98.000000 89.000000 +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 591 : avg iteration time 47.12 seconds +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 591 : data loading time 0.00 +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 591 : avg forward pass time 12.64 +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 591 : backward pass time 10.13 +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 591 : optimizer step time 1.56 +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 591-------------------- +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:23:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617340 41.706776 41.577286 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.749512 67.865234 67.402344 +util 94.000000 99.000000 90.000000 +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 592 : avg iteration time 48.61 seconds +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 592 : data loading time 0.00 +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 592 : avg forward pass time 13.37 +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 592 : backward pass time 10.15 +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 592 : optimizer step time 1.57 +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 592-------------------- +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:24:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617644 41.708008 41.577286 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.803223 67.865234 67.617188 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 593 : avg iteration time 46.11 seconds +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 593 : data loading time 0.00 +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 593 : avg forward pass time 12.13 +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 593 : backward pass time 10.15 +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 593 : optimizer step time 1.55 +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 593-------------------- +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:24:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617640 41.707985 41.577290 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.803223 67.865234 67.617188 +util 96.250000 100.000000 89.000000 +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 594 : avg iteration time 43.38 seconds +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 594 : data loading time 0.00 +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 594 : avg forward pass time 10.76 +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 594 : backward pass time 10.15 +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 594 : optimizer step time 1.57 +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 594-------------------- +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:25:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617646 41.708012 41.577290 +peak_gpu_mem_gb 101.090972 101.132214 101.077224 +peak_gpu_mem_reserved_gb 107.585449 107.589844 107.583984 +util 59.250000 65.000000 54.000000 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 595 : avg iteration time 160.89 seconds +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 595 : data loading time 0.00 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 595 : avg forward pass time 71.86 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 595 : backward pass time 8.40 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 595 : optimizer step time 0.40 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 595-------------------- +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5527 iter count: 1.0 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5527 iter count: 1.0 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5527 iter count: 1.0 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5527iter count: 1.0 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:28:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617525 41.707497 41.577290 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.125488 68.314453 67.558594 +util 96.250000 99.000000 93.000000 +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 596 : avg iteration time 44.47 seconds +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 596 : data loading time 0.00 +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 596 : avg forward pass time 11.32 +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 596 : backward pass time 10.14 +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 596 : optimizer step time 1.57 +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 596-------------------- +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:29:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617520 41.707497 41.577290 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.125488 68.314453 67.558594 +util 91.000000 96.000000 87.000000 +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 597 : avg iteration time 43.33 seconds +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 597 : data loading time 0.00 +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 597 : avg forward pass time 10.74 +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 597 : backward pass time 10.14 +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 597 : optimizer step time 1.57 +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 597-------------------- +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:29:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617520 41.707497 41.577290 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 68.101074 68.333984 67.402344 +util 97.250000 100.000000 94.000000 +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 598 : avg iteration time 48.67 seconds +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 598 : data loading time 0.00 +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 598 : avg forward pass time 13.40 +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 598 : backward pass time 10.14 +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 598 : optimizer step time 1.57 +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 598-------------------- +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:30:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.617556 41.707603 41.577290 +peak_gpu_mem_gb 65.071996 65.113239 65.058249 +peak_gpu_mem_reserved_gb 67.974121 68.314453 66.953125 +util 92.500000 99.000000 88.000000 +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 599 : avg iteration time 48.71 seconds +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 599 : data loading time 0.00 +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 599 : avg forward pass time 13.41 +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 599 : backward pass time 10.14 +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 599 : optimizer step time 1.57 +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 599-------------------- +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 01:31:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 01:31:24 | INFO | fastgen.trainer:run:225 ] Training complete. +[May 5, 2026 - 01:31:24 | INFO | fastgen.trainer:validate:358 ] Validation iteration 600 +[May 5, 2026 - 01:31:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:31:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 5, 2026 - 01:31:29 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.465 +[May 5, 2026 - 01:31:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:31:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:31:36 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.706 +[May 5, 2026 - 01:31:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:31:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:31:41 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.688 +[May 5, 2026 - 01:31:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:31:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:31:46 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.803 +[May 5, 2026 - 01:31:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:31:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:31:51 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=3.868 +[May 5, 2026 - 01:31:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:31:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:31:56 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.980 +[May 5, 2026 - 01:31:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:31:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:32:01 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.378 +[May 5, 2026 - 01:32:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:32:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:32:06 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.369 +[May 5, 2026 - 01:32:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:32:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:32:11 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.429 +[May 5, 2026 - 01:32:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 01:32:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 01:32:16 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.113 +[May 5, 2026 - 01:32:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 600-------------------- +[May 5, 2026 - 01:32:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 01:32:33 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 600 +[May 5, 2026 - 01:32:33 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.280 (n=10) +[May 5, 2026 - 01:32:33 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 600 +[May 5, 2026 - 01:32:33 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints/0000600 +[May 5, 2026 - 01:32:52 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 600 +[May 5, 2026 - 01:32:53 | INFO | fastgen.trainer:run:232 ] Taking a 10 sec nap and exiting training. +[May 5, 2026 - 01:33:03 | INFO | fastgen.utils.distributed:clean_up:88 ] Distributed clean up. +[May 5, 2026 - 01:33:03 | INFO | __main__::49 ] Training finished. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c85b7551c7425d6082c64d95baf05273c33a2b26 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/wandb-metadata.json @@ -0,0 +1,70 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-04T04:51:22.023532Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_2step_t624_no_reward.py", + "-", + "trainer.resume=False", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_2step_t624_no_reward", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1175943299072" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "btht0sri7vddjdptc2alnflsboni5uhl" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8f80a51d48eba1c5018218423d4270a577327e65 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/files/wandb-summary.json @@ -0,0 +1 @@ +{"fake_score/trainable_params":1421383840,"_wandb":{"runtime":42100},"val0/total_loss":0,"val0/sync_c_sample_5":6.979977607727051,"_step":600,"fake_score/total_params":1421383840,"val0/generated":{"_type":"videos","count":10,"videos":[{"caption":"sync_c=6.465","_type":"video-file","sha256":"d3b3c38cc4cea459a9ba1c021895364ca9ae872202d870a9df2e11f5fe734db0","size":213374,"path":"media/videos/val0/generated_600_d3b3c38cc4cea459a9ba.mp4"},{"size":214086,"path":"media/videos/val0/generated_600_1b22e3065fc30b1fc3cb.mp4","caption":"sync_c=5.706","_type":"video-file","sha256":"1b22e3065fc30b1fc3cbeb5167f60ec66c005e488d63f4dec82349b8cbc26012"},{"path":"media/videos/val0/generated_600_e305c2ec1a356386ea3a.mp4","caption":"sync_c=5.688","_type":"video-file","sha256":"e305c2ec1a356386ea3a46601e061d7c700229b33d09a7e02e7b462110835147","size":208137},{"size":294516,"path":"media/videos/val0/generated_600_3c513fbe04eea74399aa.mp4","caption":"sync_c=4.803","_type":"video-file","sha256":"3c513fbe04eea74399aa394a92545729fab7216e103ecdae2fd35a6dcc58b645"},{"sha256":"c7944e8bf03d0b8b4341e0d7bbee5f4e7b0adc3d3330765fb30f75d0d5357645","size":255214,"path":"media/videos/val0/generated_600_c7944e8bf03d0b8b4341.mp4","caption":"sync_c=3.868","_type":"video-file"},{"_type":"video-file","sha256":"134b51494130bcd535cf515bed310a633a8b72c42f368eeb1862567657d0e0ef","size":184559,"path":"media/videos/val0/generated_600_134b51494130bcd535cf.mp4","caption":"sync_c=6.980"},{"caption":"sync_c=6.378","_type":"video-file","sha256":"00cb74bae01fa7368a462766cf019fa01052af74b46f580f9b45b74664db14e0","size":294125,"path":"media/videos/val0/generated_600_00cb74bae01fa7368a46.mp4"},{"size":198675,"path":"media/videos/val0/generated_600_c8e42f6e6ca899b7d68d.mp4","caption":"sync_c=4.369","_type":"video-file","sha256":"c8e42f6e6ca899b7d68d4632aaad57bff35f1e5e8dee2c8bdc68d1a34ff2c5b7"},{"caption":"sync_c=5.429","_type":"video-file","sha256":"96c9100dc7b878101ec0ab5f2a75fda367ee6f900907e3fb2cc2d2e03252e4a8","size":240915,"path":"media/videos/val0/generated_600_96c9100dc7b878101ec0.mp4"},{"path":"media/videos/val0/generated_600_3e07d13ddaace09f352a.mp4","caption":"sync_c=3.113","_type":"video-file","sha256":"3e07d13ddaace09f352a05693b03db33cbb4b216740402c34f83c9f6206c14aa","size":245698}],"captions":["sync_c=6.465","sync_c=5.706","sync_c=5.688","sync_c=4.803","sync_c=3.868","sync_c=6.980","sync_c=6.378","sync_c=4.369","sync_c=5.429","sync_c=3.113"]},"val0/sync_c_sample_8":5.429076194763184,"profiler/backward_pass_time":10.144969090819359,"train/gan_loss_gen":0,"optimizer/lr_net":2e-06,"model/total_params":17136849376,"_timestamp":1.7779123533733978e+09,"val0/sync_c_sample_0":6.465003490447998,"train/gan_loss_disc":0,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"optimizer/non_finite_grads_count (model_key net)":0,"train/total_loss":0.0097198486328125,"train_media/data/real":{"_type":"video-file","sha256":"2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae","size":190461,"path":"media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4"},"profiler/data_loading_time":0.0011148042976856232,"val0/sync_c_sample_6":6.377812385559082,"net/trainable_params":1421383840,"val0/reconstructed":{"videos":[{"_type":"video-file","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028,"path":"media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4"},{"size":170922,"path":"media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4","_type":"video-file","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214"},{"size":173236,"path":"media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4","_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8"},{"path":"media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4","_type":"video-file","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372},{"path":"media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4","_type":"video-file","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004},{"_type":"video-file","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630,"path":"media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4"},{"_type":"video-file","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986,"path":"media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4"},{"_type":"video-file","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516,"path":"media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4"},{"_type":"video-file","sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001,"path":"media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4"},{"sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490,"path":"media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4","_type":"video-file"}],"captions":false,"_type":"videos","count":10},"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"model/local_trainable_params":[711111248,710552144,710552144,710552144],"train/vsd_loss_unweighted":0.552734375,"train/fake_score_loss":0.0097198486328125,"val0/sync_c_mean":5.279874563217163,"profiler/avg_iteration_time":48.70817279815674,"val0/sync_c_sample_7":4.368593215942383,"optimizer/lr_fake_score":2e-06,"val0/sync_c_sample_2":5.688401222229004,"profiler/avg_forward_pass_time":13.412094060331583,"val0/sync_c_sample_4":3.867547035217285,"_runtime":42100.178562604,"profiler/optimizer_step_time":1.5731049813330173,"train/vsd_loss":0.552734375,"model/local_total_params":[4285560952,4283762808,4283762808,4283762808],"val0/sync_c_sample_3":4.803318023681641,"net/local_total_params":[355555624,355276072,355276072,355276072],"train/vsd_loss_weighted":0.552734375,"net/total_params":1421383840,"val0/sync_c_sample_9":3.113408088684082,"train_media/student/generation":{"size":213362,"path":"media/videos/train_media/student/generation_500_cfcf07eca45d7088f701.mp4","_type":"video-file","sha256":"cfcf07eca45d7088f701b448ea711950d0a54a6f3ef852b62eea2308234c436c"},"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"optimizer/iteration":600,"model/trainable_params":2842767680,"optimizer/grad_norm (model_key net)":0,"val0/sync_c_sample_1":5.705608367919922} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6a45b83eb16898f329080d4585eee144e176e400 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-04T13:51:22.399551704+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpfqt_ksxp/port-493121.txt","pid":493121,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-04T13:51:22.400799537+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":493121} +{"time":"2026-05-04T13:51:22.400793987+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-493121-509664-3527652850/socket","Net":"unix"}} +{"time":"2026-05-04T13:51:22.510925872+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-04T13:51:22.529247695+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"b1ilcpvk","id":"1(@)"} +{"time":"2026-05-04T13:51:23.093349985+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"b1ilcpvk","id":"1(@)"} +{"time":"2026-05-04T13:51:29.251602533+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"mqi86wj93v67"} +{"time":"2026-05-05T01:33:03.996331823+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T01:33:03.996511277+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T01:33:03.996505627+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T01:33:03.996566663+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T01:33:03.996588492+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-493121-509664-3527652850/socket","Net":"unix"}} +{"time":"2026-05-05T01:33:05.472257985+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-05T01:33:05.472277076+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-05T01:33:05.472289599+09:00","level":"INFO","msg":"server is closed"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..90a70ee67d959267ccdc9cc04b348ad81d229a59 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-04T13:51:22.529921257+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-04T13:51:23.09264564+09:00","level":"INFO","msg":"stream: created new stream","id":"b1ilcpvk"} +{"time":"2026-05-04T13:51:23.093286905+09:00","level":"INFO","msg":"handler: started","stream_id":"b1ilcpvk"} +{"time":"2026-05-04T13:51:23.093343757+09:00","level":"INFO","msg":"stream: started","id":"b1ilcpvk"} +{"time":"2026-05-04T13:51:23.093361998+09:00","level":"INFO","msg":"sender: started","stream_id":"b1ilcpvk"} +{"time":"2026-05-04T13:51:23.093384534+09:00","level":"INFO","msg":"writer: started","stream_id":"b1ilcpvk"} +{"time":"2026-05-05T01:33:03.996515354+09:00","level":"INFO","msg":"stream: closing","id":"b1ilcpvk"} +{"time":"2026-05-05T01:33:05.171530649+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T01:33:05.471966099+09:00","level":"INFO","msg":"handler: closed","stream_id":"b1ilcpvk"} +{"time":"2026-05-05T01:33:05.472037644+09:00","level":"INFO","msg":"sender: closed","stream_id":"b1ilcpvk"} +{"time":"2026-05-05T01:33:05.472054626+09:00","level":"INFO","msg":"stream: closed","id":"b1ilcpvk"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f1b97c94f78f7f5a9877153f51be6bb0b52e2281 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_setup.py:_flush():81] Configure stats pid to 493121 +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug.log +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/logs/debug-internal.log +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_init.py:init():844] calling init triggers +2026-05-04 13:51:22,024 INFO MainThread:493121 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_2step_t624_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '600', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-04 13:51:22,025 INFO MainThread:493121 [wandb_init.py:init():892] starting backend +2026-05-04 13:51:22,510 INFO MainThread:493121 [wandb_init.py:init():895] sending inform_init request +2026-05-04 13:51:22,524 INFO MainThread:493121 [wandb_init.py:init():903] backend started and connected +2026-05-04 13:51:22,528 INFO MainThread:493121 [wandb_init.py:init():973] updated telemetry +2026-05-04 13:51:22,557 INFO MainThread:493121 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-04 13:51:23,816 INFO MainThread:493121 [wandb_init.py:init():1042] starting run threads in backend +2026-05-04 13:51:24,244 INFO MainThread:493121 [wandb_run.py:_console_start():2524] atexit reg +2026-05-04 13:51:24,244 INFO MainThread:493121 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-04 13:51:24,244 INFO MainThread:493121 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-04 13:51:24,244 INFO MainThread:493121 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-04 13:51:24,262 INFO MainThread:493121 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 01:33:03,996 INFO wandb-AsyncioManager-main:493121 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 01:33:03,996 INFO wandb-AsyncioManager-main:493121 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/run-b1ilcpvk.wandb b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/run-b1ilcpvk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1a2ea8c31a7f4e38996dfe58644f780a9820f0ef --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb/run-20260504_135122-b1ilcpvk/run-b1ilcpvk.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f6ebcffecc0034e744374ac6a82c524532fdf79ecfd79088a941034730a7722 +size 9303560 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb_id.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..048b6cebce06663edb27915c18bd2072faf8d39c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T624_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t624_no_reward/wandb_id.txt @@ -0,0 +1 @@ +b1ilcpvk diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..427a2f3b9afe348599c52fedbc28efc1b5524b47 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_2step_t937_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'False' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '0' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.937' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '100' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7c35a66ad348a1e72468e8a58aa0040bcb802ddd --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T16:24:36.432849483+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T16:24:36.919082858+09:00","level":"INFO","msg":"stream: created new stream","id":"9ojgh41i"} +{"time":"2026-05-05T16:24:36.919723346+09:00","level":"INFO","msg":"handler: started","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:24:36.919786524+09:00","level":"INFO","msg":"stream: started","id":"9ojgh41i"} +{"time":"2026-05-05T16:24:36.919804273+09:00","level":"INFO","msg":"sender: started","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:24:36.919830156+09:00","level":"INFO","msg":"writer: started","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:33:51.211321812+09:00","level":"INFO","msg":"stream: closing","id":"9ojgh41i"} +{"time":"2026-05-05T16:33:51.810152536+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T16:33:52.113831777+09:00","level":"INFO","msg":"handler: closed","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:33:52.113883038+09:00","level":"INFO","msg":"sender: closed","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:33:52.113898223+09:00","level":"INFO","msg":"stream: closed","id":"9ojgh41i"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d79de7ae7cb1ab2b8d25d1a9c40bef3a92298485 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/debug.log @@ -0,0 +1,21 @@ +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_setup.py:_flush():81] Configure stats pid to 629855 +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug.log +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug-internal.log +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:init():844] calling init triggers +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_2step_t937_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '100', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:init():892] starting backend +2026-05-05 16:24:36,412 INFO MainThread:629855 [wandb_init.py:init():895] sending inform_init request +2026-05-05 16:24:36,427 INFO MainThread:629855 [wandb_init.py:init():903] backend started and connected +2026-05-05 16:24:36,431 INFO MainThread:629855 [wandb_init.py:init():973] updated telemetry +2026-05-05 16:24:36,452 INFO MainThread:629855 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 16:24:37,792 INFO MainThread:629855 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 16:24:38,267 INFO MainThread:629855 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 16:24:38,267 INFO MainThread:629855 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 16:24:38,267 INFO MainThread:629855 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 16:24:38,267 INFO MainThread:629855 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 16:24:38,275 INFO MainThread:629855 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 16:33:51,211 INFO wandb-AsyncioManager-main:629855 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 16:33:51,211 INFO wandb-AsyncioManager-main:629855 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adbc89d64da54c8889f9eebbccd6673f41a32ec7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/config.yaml @@ -0,0 +1,390 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + wqjrzhjrn5xgtnhk7eb06n2h8zcma2xp: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_2step_t937_no_reward.py + - '-' + - trainer.resume=False + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_2step_t937_no_reward + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=100 + - trainer.save_ckpt_iter=100 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1176060174336" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward + startedAt: "2026-05-05T07:24:35.920029Z" + writerId: wqjrzhjrn5xgtnhk7eb06n2h8zcma2xp + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_2step_t937_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "False" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "0" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.937" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "100" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/output.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..fcc7f8e252547d0a6f53c3c27f91014f82386939 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/output.log @@ -0,0 +1,169 @@ +[May 5, 2026 - 16:24:38 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 5, 2026 - 16:24:38 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 5, 2026 - 16:24:38 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 5, 2026 - 16:24:38 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 5, 2026 - 16:24:38 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 5, 2026 - 16:24:38 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 5, 2026 - 16:24:38 | INFO | fastgen.trainer:run:77 ] Starting training +[May 5, 2026 - 16:24:38 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 5, 2026 - 16:24:38 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 5, 2026 - 16:25:04 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 5, 2026 - 16:25:15 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 5, 2026 - 16:25:15 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 5, 2026 - 16:25:15 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 5, 2026 - 16:25:15 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 5, 2026 - 16:25:59 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 5, 2026 - 16:25:59 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 5, 2026 - 16:25:59 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 5, 2026 - 16:25:59 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 5, 2026 - 16:25:59 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 5, 2026 - 16:25:59 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 5, 2026 - 16:26:01 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 16:26:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 5, 2026 - 16:26:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 5, 2026 - 16:26:02 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 16:26:03 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 16:26:03 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.9s +[May 5, 2026 - 16:26:03 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 5, 2026 - 16:26:03 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 16:26:15 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 16:26:16 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.3s +[May 5, 2026 - 16:26:16 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 5, 2026 - 16:26:16 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 5, 2026 - 16:26:16 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 5, 2026 - 16:26:16 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[May 5, 2026 - 16:26:16 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 5, 2026 - 16:26:22 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 5, 2026 - 16:26:22 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 5, 2026 - 16:26:22 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 5, 2026 - 16:26:22 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17136.85 M total params (logical). +[May 5, 2026 - 16:26:22 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4285.56 M total params LOCAL on rank 0. +[May 5, 2026 - 16:26:22 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 1. +[May 5, 2026 - 16:26:22 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 2. +[May 5, 2026 - 16:26:22 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 3. +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 16:26:23 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 16:26:23 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.337015 37.382362 37.252399 +peak_gpu_mem_gb 57.011160 57.011160 57.011160 +peak_gpu_mem_reserved_gb 59.156250 59.156250 59.156250 +util 80.000000 85.000000 76.000000 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.46 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 10.38 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 10.12 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.78 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:27:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.512817 37.558968 37.428204 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 91.500000 97.000000 87.000000 +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 45.90 seconds +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.01 +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 10.16 +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 1.58 +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:27:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.513610 37.559486 37.428345 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.191895 61.193359 61.191406 +util 95.250000 99.000000 92.000000 +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 49.05 seconds +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 13.55 +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 10.18 +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 1.57 +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:28:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.514299 37.562130 37.428368 +peak_gpu_mem_gb 59.658712 59.660274 59.658191 +peak_gpu_mem_reserved_gb 61.211426 61.212891 61.210938 +util 95.000000 99.000000 88.000000 +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 43.45 seconds +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 10.75 +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 10.18 +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 1.58 +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:29:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.535754 37.584053 37.450436 +peak_gpu_mem_gb 96.121632 96.123194 96.121111 +peak_gpu_mem_reserved_gb 102.568848 102.658203 102.539062 +util 60.000000 65.000000 53.000000 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 161.44 seconds +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 71.85 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 8.46 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.44 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.8301 iter count: 1.0 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8301 iter count: 1.0 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.8301 iter count: 1.0 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8301iter count: 1.0 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:32:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.535722 37.584145 37.450550 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.758789 64.820312 64.738281 +util 96.000000 99.000000 89.000000 +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 49.97 seconds +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 14.04 +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 10.16 +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 1.57 +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:32:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.536128 37.584496 37.450550 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656250 64.703125 64.640625 +util 96.250000 100.000000 90.000000 +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 48.66 seconds +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 13.39 +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 10.13 +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 1.57 +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:33:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3af2978ef3a921c2d2aaa618ff3963a65e581d39 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-05T07:24:35.920029Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_2step_t937_no_reward.py", + "-", + "trainer.resume=False", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_2step_t937_no_reward", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=100", + "trainer.save_ckpt_iter=100" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1176060174336" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "wqjrzhjrn5xgtnhk7eb06n2h8zcma2xp" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ac01ef2a79862ad6dd6a9214b48a846d505bf159 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/gan_loss_gen":0,"profiler/data_loading_time":0.0007740277796983719,"_runtime":553.416789332,"profiler/optimizer_step_time":1.5734902396798134,"optimizer/lr_fake_score":2e-06,"net/local_total_params":[355555624,355276072,355276072,355276072],"profiler/avg_forward_pass_time":13.394969936460257,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"train/total_loss":0.00679779052734375,"net/trainable_params":1421383840,"optimizer/non_finite_grads_count (model_key net)":0,"model/local_trainable_params":[711111248,710552144,710552144,710552144],"fake_score/total_params":1421383840,"_step":7,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"train/vsd_loss_unweighted":0.830078125,"model/total_params":17136849376,"optimizer/lr_net":2e-06,"_timestamp":1.777966428137382e+09,"profiler/avg_iteration_time":48.6568386554718,"train/vsd_loss":0.830078125,"train/fake_score_loss":0.00679779052734375,"profiler/backward_pass_time":10.127559842541814,"net/total_params":1421383840,"train/vsd_loss_weighted":0.830078125,"model/trainable_params":2842767680,"train/gan_loss_disc":0,"_wandb":{"runtime":553},"model/local_total_params":[4285560952,4283762808,4283762808,4283762808],"fake_score/trainable_params":1421383840,"optimizer/grad_norm (model_key net)":0,"optimizer/iteration":7,"net/local_trainable_params":[355555624,355276072,355276072,355276072]} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..a603954af516fd73f5fa2de26701dfce746bbba0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2026-05-05T16:24:36.293229481+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpkpubqri2/port-629855.txt","pid":629855,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-05T16:24:36.295334885+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":629855} +{"time":"2026-05-05T16:24:36.29532374+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-629855-651385-4078189663/socket","Net":"unix"}} +{"time":"2026-05-05T16:24:36.413434619+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-05T16:24:36.432275132+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"9ojgh41i","id":"1(@)"} +{"time":"2026-05-05T16:24:36.919792703+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"9ojgh41i","id":"1(@)"} +{"time":"2026-05-05T16:24:43.274532387+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"rk2vpho8rppr"} +{"time":"2026-05-05T16:33:51.211272354+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T16:33:51.211321615+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T16:33:51.211316128+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T16:33:51.211364979+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T16:33:51.21137622+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-629855-651385-4078189663/socket","Net":"unix"}} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7c35a66ad348a1e72468e8a58aa0040bcb802ddd --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T16:24:36.432849483+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T16:24:36.919082858+09:00","level":"INFO","msg":"stream: created new stream","id":"9ojgh41i"} +{"time":"2026-05-05T16:24:36.919723346+09:00","level":"INFO","msg":"handler: started","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:24:36.919786524+09:00","level":"INFO","msg":"stream: started","id":"9ojgh41i"} +{"time":"2026-05-05T16:24:36.919804273+09:00","level":"INFO","msg":"sender: started","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:24:36.919830156+09:00","level":"INFO","msg":"writer: started","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:33:51.211321812+09:00","level":"INFO","msg":"stream: closing","id":"9ojgh41i"} +{"time":"2026-05-05T16:33:51.810152536+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T16:33:52.113831777+09:00","level":"INFO","msg":"handler: closed","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:33:52.113883038+09:00","level":"INFO","msg":"sender: closed","stream_id":"9ojgh41i"} +{"time":"2026-05-05T16:33:52.113898223+09:00","level":"INFO","msg":"stream: closed","id":"9ojgh41i"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d79de7ae7cb1ab2b8d25d1a9c40bef3a92298485 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_setup.py:_flush():81] Configure stats pid to 629855 +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug.log +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/logs/debug-internal.log +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:init():844] calling init triggers +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_2step_t937_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '100', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 16:24:35,921 INFO MainThread:629855 [wandb_init.py:init():892] starting backend +2026-05-05 16:24:36,412 INFO MainThread:629855 [wandb_init.py:init():895] sending inform_init request +2026-05-05 16:24:36,427 INFO MainThread:629855 [wandb_init.py:init():903] backend started and connected +2026-05-05 16:24:36,431 INFO MainThread:629855 [wandb_init.py:init():973] updated telemetry +2026-05-05 16:24:36,452 INFO MainThread:629855 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 16:24:37,792 INFO MainThread:629855 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 16:24:38,267 INFO MainThread:629855 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 16:24:38,267 INFO MainThread:629855 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 16:24:38,267 INFO MainThread:629855 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 16:24:38,267 INFO MainThread:629855 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 16:24:38,275 INFO MainThread:629855 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 16:33:51,211 INFO wandb-AsyncioManager-main:629855 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 16:33:51,211 INFO wandb-AsyncioManager-main:629855 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/run-9ojgh41i.wandb b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/run-9ojgh41i.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a06350898b437cd0cbcb3d4fc9bca29e386bdd37 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb/run-20260505_162435-9ojgh41i/run-9ojgh41i.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31ee93a7f7ff1255ac3fb8cf05cd74028c7a64be43de92071025d7459b543bc5 +size 134722 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb_id.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..ca530959fe3ee6d28c2e0a1d665a23a9298fd41b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_2STEP_T937_NO_REWARD/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_2step_t937_no_reward/wandb_id.txt @@ -0,0 +1 @@ +9ojgh41i diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_BRANCH_700_TO_800_FRESHOPT/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_700to800_freshopt/checkpoints/0000700.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_BRANCH_700_TO_800_FRESHOPT/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_700to800_freshopt/checkpoints/0000700.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e4ae24450baf0c9812a998397c11a4f9073a0b2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_BRANCH_700_TO_800_FRESHOPT/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_700to800_freshopt/checkpoints/0000700.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9f2452dddd8f3738f9d41f95cde5ea79644eb22f14c88132582bba152f3bd53 +size 1121714176 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_BRANCH_700_TO_800_FRESHOPT/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_700to800_freshopt/checkpoints/0000700.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_BRANCH_700_TO_800_FRESHOPT/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_700to800_freshopt/checkpoints/0000700.pth new file mode 100644 index 0000000000000000000000000000000000000000..65e0c235adb1ae0e7b416e74128bff2cacc74901 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_BRANCH_700_TO_800_FRESHOPT/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_700to800_freshopt/checkpoints/0000700.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9681d86b2c640925539ad92ca52297f133f17cce627698baaca064df80c997f4 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6c34edd3d8585e934c0391038e7d1339d6896810 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ebd96da76380d09ad7da826875ca48a50fe52be87cfc521c6f806999c574278 +size 635521 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..596337083608c1641b1bf613da6dab59218ea55c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58bde1b7a53ef209bef6825139c269cda29e6d1ef57bf49453bdbe736759be88 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9107db18d1050b90d4d9ed20a516a88659f4f759 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae44916aab01930b59286b5c8cc6c8f8da0d2225c7d67e355759cb0f8becd338 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9227e656bff4260557d0546031c8f1ea0ddacb69 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeafeb5ab8bbc444596671ddcac5787e9b0e8678261fe5ffa6612334a873a62 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..101ba9e705d0c0d27ffb2021e4ee1aba7d6296f4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eda90de6a86b7ac1470c4caa4e49d5d160c965ced0849d4cb5bc604a0f850035 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1ccf324574d4e91c5f93fdfe05bf8947e55e8b5a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c49b1425e0aeb56ba2570d67fc8678deea682c1cfb3f2e24804c2a67b52491d9 +size 635514 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fbba0989e1a5d6c999630f18dc00a6b7a765a1ad --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61163e4d37bf88c67637a66f0f80ba92e8d0bcb1457dd9f6f17d0c0185c4352f +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f74a1e236525c9574869404a305a1a5052fdce48 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66d41b437fc5b0846c7946a225fee4ecbef89c5efbfd7cdc246eeb86922b8855 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..711c8fece74ae0cbba064b91bb207302fb324031 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00f12e56b4c91f31071b118629e7d9794cba62824146c66a0c5ef698c472ac9d +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..932188c9cab741691ae0b460bc5d058e712ea0ff --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a53a7aa11be9ec395b30a7faa0dc3fd055a1fdefad49398d913a58e433119a +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..df0fb483027e40ac33d6a36f4b38fcfd244d6a44 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2110994f1f90ec15f3f831251e70d29fe1b83f8a35bb969069c4ac3f211c6d +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..ad27e5f7eea4205b2bef82e482d076aaade7963b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa66b65c62e7f197a9cbb9d17b137faca6d1b083d2e350d8d869c43906af0161 +size 635530 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8a683d013d123a3011a1c538b22ef93249316501 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:031dbc9010765d2161501af44a99a468a3886c027c8a842de7f7bb308dd90bcd +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6918b2ea30272426477ca79bb5b03c617d315c2a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e6379cd92381bf77d9402c139a2ec7b7f8da128475f0c3c0d8550ff04e33eea +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e49c8d3d43d58ae07fffbea2bdf40d5b162c447a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ac10a74638930d6549068b9610821c6fb4e649bffd2fde9308c45621aea3d2 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4ba29221b5613e3ec0c265fcfa429062683c0571 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3a703aea84c29a28b83bf6405006fed86520a490c011be7c7eeb135ef4126b5 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9cadfeece2e874e7ddac33721fcb8a9d85909c62 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c425874a1ac7f48b6c0ba4d0a87979e9b6e3405278c669423b02497ea1a46a +size 635523 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8d9ac842a981dbec7ca6077b23b43ff5a062b9d5 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b64cf10a4c399fd4e454df598d411a25dce11b2a572d4b64e01f21f8cea3ba +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8da7b96f2ab28434a68346b704824556e45602f8 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:564e8461191bd407427af1ed38eacfc5bb4ef331b0911a29dc84b7b49cfdf55a +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0f2d43180cbac3730c4416c28223cbc1d06d3d89 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1876403f0118508a679d507fc5b7223aa8e38f18fb3d97b27a06f126f4ef2bfa +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..db73821bdd4d4b880ace393e647acf7f3cdb04a6 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:759b3be88e484658b410ad4c1b74632a2e6e3680ff1a3b51c58d5e12d229467d +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.pth new file mode 100644 index 0000000000000000000000000000000000000000..130c61950c21828925cda2c16a5be77c70be72e4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c097cd2564e2f877acb5e00b0b08cbb2ce9c3d3500939503b1bf26d185a049e +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f8195a7e832ed0ea866c9219df17d5fde3fa7504 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b004e47e001a72ad30fdacef5d0e9c9685cc5fdef12f49c77998380d4defa1c6 +size 635530 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6792546c3d1597b6b7b3e520f7f01ccf341391b0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bf2fb56bdccf4ca87b7e766abbcefbd91e20c5ec0eece6004259a7f96fc76cf +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0044b4c019760f8795e009f63dec5ec2e81cd7a9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e11d417bfbc2f1fa2a90fda9896aa1ad6436414eb35f036d20c15ec76abe585e +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..277dda2a5ffc9e6f4a14d1b8038664190317176f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3e3811993398d55742e198fdcdebf7936b0b1a7b21b8e3653b7d9e4e8b84667 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6faa05ce46f3cb456315fe422498373506abcf12 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:532a0f4d53904fc19eb0c3a6a7e6164302c453299b6c1992cefb64375b0daee7 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c0f5a05eda90671874fc55323add537e0fda2acf --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19689efd8ce7a558f7786a67bc1c1591f323adb4025e764e5b17294ae98a925f +size 635523 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2aba7ad456b41154e30a66db20bf0d2d38e3eb2a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d5ab9eaa6b9ceeb05a4165f23d09ad018ad8b4d05b4b3946e6754d49feabdbe +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c906bda2817b35c8b50eb41b74b9b63a10cb0cec --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbf190786ea536e5e14c064613130e39ee1aea17ef18e7449ed628a8cb10e3ba +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9b81ecf039491e94e73ec508ad4830b9853d0068 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b76725d8b12d618e5f54d9d4546fed32983d1ca4aff7eb7b060f650e0ee9ce8 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9bfffade564300135fe173aea5786c46b724c14d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccbe5c1fe1d16a32e54c02164878a8c05c5a0fb29625eef7b87ce3451ac69390 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.pth new file mode 100644 index 0000000000000000000000000000000000000000..65e0c235adb1ae0e7b416e74128bff2cacc74901 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9681d86b2c640925539ad92ca52297f133f17cce627698baaca064df80c997f4 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..de51a132c256b02990400044c1e96c898fcc4711 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70d109212efa01ce7971fc3bd7df00f8267cfc4bc8f016c00ab44c4d205cbbca +size 635530 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..20e7f09530444e9528eead439e832685ffa245dd --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06cfa6a23f6444bd52e37d9f47c2c3896110c634dcfa6b858194a7046713592c +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b412c192efcbb3c42d423f3e18342d5d3db0c83c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fb0363cf9a1f9fd9e9c674d4b9e4772e77c5eade50c467fa86d060db5904ded +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bd1526098ab531f363aa0db719df92ddb8f5f4fe --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09fba90d6557ed2d09a27ac6e1366c1b4adc4c8a08ad81b0a6a4e90adc3a2b7a +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..784430403affbad015fb29662e9e78bd079c64f5 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4ec3eb551da21f7a314dcb736ceb9758e8f68b1316ddb00acf2d7c152af6a18 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..a655b637599b5868532a2f799479c84e9d4c90f9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb2f71d74b537d2e29844eb3bc29b061483b93c037f3724393ceff03f1f87232 +size 635523 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..53c1a58b28929e269c8201e81dea66f7ee47d8db --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa5e5b3bce4b65f04f93c0e2f945e0dc0c7aa74c8f6d24b931f3f468eb81c037 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..baa549e02103c88109e095a15bddd58f8666ad11 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d03f2cfc0e412651e0eb319b7ea77bde712145bf09c52a4eec1b827634e03807 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..84c6be2781f7d7484344c6b292db542a240f9d8b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f93c12a150392e50e5305edf0df8db0f9d7e7aff66994a39cb5b4f5e063ccfe7 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bd22737e71f036e70afdd8f13a2513d4db601182 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03a0a706e5d264b6e32acf8a5d8ba96739706f2daf213ae453d8de7d7783ac8 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.pth new file mode 100644 index 0000000000000000000000000000000000000000..d9428fd0bee1a262cb6c85a97fe5ad137b75abf3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2dc0baa0ad04475f065f020e0b079278e3e62139effea8abc09a5b94492e991 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..18fae0f0c6cda32359bd71c8826a068beeabf792 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:525b51eefec9667d16df75ff1e2d0804aa9634f33adada9973bbe0476fbed19d +size 635530 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a4cc3230e9c66d4b6c14bc3d4c0290a9d6e539bc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee96a7ad53f808d47788773db04389e34896da0db6d2be1f30e687b4bad10120 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..363fd40871c5b8f823c6f3130a15cb0c10ccb097 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df4201c82426e7366078e66c9182d5520784c06c82dd08155606e44f4116af94 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..040903622eda0779937cda96a9fe217848297273 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa861db1c527a3f30ac8405dfcd955a6c1c76cec97c13bb00e4075aa475fe5c +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..55ef688d9b5a35c4fe89970f73dad79b458fce90 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3eeb0a9976ce1650d21a5f1fa793330958ad369eccfa03821d93867f40a5883 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d21f92699c6cf11084b463f151daa7365eb310f3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39b609e3c4aafb689cf9a98c80e0ab5e14c1307b8494567079591fa07214ace +size 635523 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0142fc19f42ba9a96a2066dd9703e1e96ba53da5 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f15b643ff200a62ede78cf7610c8792a1920aca0250e1b95af92e8bdc3fbb2a +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..723528cb87596d7b7d8b26103a2722c7d2a6ad32 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e22ccb408e9cbdaee804d7a819a0905066caf1e1c135fc4977682e4f99237197 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8ee7b69a7f56fffd6989e4fde5cdd00103295191 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f41e6fd0504ff7be3d8bc9737c8895819a567069c5674fe628c31d283a48ab3 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4372d1c79c251624f2cf5fae06bad5c90ac8264c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a37c31f9294020a61a7d1c49e689368e860f693369de1cb83a04563f8d5a6ce6 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.pth new file mode 100644 index 0000000000000000000000000000000000000000..cc3a2abee3ecbd789087be5230951517803c0f45 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d45519548259bfe1c9514c2ebb15be7684405899b3c8a6692a0a5ed71c8d6b68 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..53122d17c43f4c690f45f8c72fab43568ad78555 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85e2e9f608c03a03b9986fb42dfb294c8209d3b05f37773685aa0c84802fd9dd +size 635530 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..806787b3f380b3a0ad07128928e9a6154c0ada27 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:520b570d99fded04b4e34a2028c2c3d9a1d9e229541e42451eba9497b96544a6 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c801f1c1dc72047c0f1bbfc177e1a9775b2e58e7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:714b3e842439394fba98b266e4c97505bf0d9ae3cb0c1ea8996d689a298ad80e +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..622cda685062a81d243e34de0a5dce6128bbc78f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dca142a467a8e2b84b000f3c56de3c05182d1e4d918e1f77d11ab69af5deea2a +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..33e1eaad0a8c5607323952f7aef31a8733967ee1 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691a962a7afc0164ec74a3f7add8c72de2512bd693e9abad6840c080b698023d +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..a7d9f9b4b2350a9f406c24983eedc25687d60282 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b79ced0897f1484d5c33ef0f299522d88d1baadade4da8da22c4661922ac319 +size 1722894 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bd460e5c665adac62b155dba77da88328a72846b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb085fc775d1e5e6ff9de876e5c8f364343ffe97ca6370a2153f9e44c43b2cf6 +size 2847149355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bf0f3d4e04aaf83fbcb63c086ade2842b7a43652 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c83eb4711ad0098a8bc47df0ba934696f68fb3f4eddad30864e40aed54f1e4b +size 2845267132 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..081ad22c8560c6fd6ef8c33272b7825941c7cbbb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70fad6977a0f2cfdfd98ba1d7337a0daf5071cda7170467fa37da8b1f9b1d4fa +size 2845352958 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e969f27f75e70bac751b6bd83c9d915ae561b9d4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9319293a7d86a9804424f410d4c52d2f78d07f6062ded17c0eaf7323d9ff31e4 +size 2845316241 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..22433f9e9442460c763f02ccc4b38ccaefdfb666 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c631ed54b3dfdb0a43dd9f50b3a145e2608c747bca042b80cc7bb349bfa01cb +size 635523 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c3367282af054aa0db6329693db2ef39aada45e9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbb286e1c0507e00f47195ac4768983becdf47158da41aaf7c7f98e319c2ca1b +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fd827e1f002e14f17ab1037ab3e2ecc816e61423 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d4f020e65fb9d76231a2e930398f21989182e90a09fcd1c9e4d3b97b7be6f21 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..711d28e18c67375c1f284551190a83541cc67bda --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7979f911d278ff17f7b17606cdf76522fce37e6b08ddfb5c1e85b690f1958df +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..85496dbd0a0f7a388579b37a4b362c970e979046 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:853262b3df0d6d7d55ac203173a3fb2ea45066a660b534297b4561582baf41cf +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..b6e8d9d40783456f0911cf547def8cff87ced9b2 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5125a16336ab5f9eb9053402283f41dcda9e93773d8011c88a53429e80d5fe88 +size 1722737 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..196efd775f378207de63b6e43c332ee53313b553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73836c005ac9d1d62cc537f7c55941aabf0bd7a3a66e8c3a8cffbae716bd9fc3 +size 2847149355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..296a0234ef2919b4466ec55c0c49fcc4c079df4e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef817ecbb942bbbfd5a355f1ee83d73bc445df216abfa5099d3922db1645234 +size 2845267132 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..163862265a674f287880001dda156464bc92ec41 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56dd5d6ef2cd2b61cd40db54ad6d77748419e6e09e3d3004ffe87fc806577182 +size 2845352958 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d7ea45f0d20515268adebafa0931b87b1f94da63 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fcbfc1c5fc116229e58894b05f6e1fd68a09909a3725af48937604358a3b0db +size 2845316241 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.pth new file mode 100644 index 0000000000000000000000000000000000000000..67cfbce89cdd30c794f3d65c737e97e90d2106a0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d72700e9be0789a64caa4b3a5e662c1e59e497d0e78ff5fb0194a2dee46e7d3 +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afaf3dd69ebe9ea17b4ba1487c0963485417a991 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'True' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '2' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.769' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '1000' + offload_module_in_decoding: 'False' + resume: 'True' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c2f4ccd8a3d708e5e5b5672f2085fb3e1a41a710 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T14:22:06.33051782+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T14:22:06.836248611+09:00","level":"INFO","msg":"stream: created new stream","id":"lvwg9pfc"} +{"time":"2026-05-05T14:22:06.83692293+09:00","level":"INFO","msg":"handler: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T14:22:06.836976277+09:00","level":"INFO","msg":"stream: started","id":"lvwg9pfc"} +{"time":"2026-05-05T14:22:06.836995755+09:00","level":"INFO","msg":"sender: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T14:22:06.837022091+09:00","level":"INFO","msg":"writer: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T16:20:15.570749087+09:00","level":"INFO","msg":"stream: closing","id":"lvwg9pfc"} +{"time":"2026-05-05T16:20:16.516503136+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T16:20:16.905948798+09:00","level":"INFO","msg":"handler: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T16:20:16.906020747+09:00","level":"INFO","msg":"sender: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T16:20:16.906038102+09:00","level":"INFO","msg":"stream: closed","id":"lvwg9pfc"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a81103634c834c7b3eb6918f322689edbdea04de --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug.log @@ -0,0 +1,22 @@ +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_setup.py:_flush():81] Configure stats pid to 264993 +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug.log +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug-internal.log +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_init.py:init():844] calling init triggers +2026-05-05 14:22:05,836 INFO MainThread:264993 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '1000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 14:22:05,836 INFO MainThread:264993 [wandb_init.py:init():892] starting backend +2026-05-05 14:22:06,310 INFO MainThread:264993 [wandb_init.py:init():895] sending inform_init request +2026-05-05 14:22:06,325 INFO MainThread:264993 [wandb_init.py:init():903] backend started and connected +2026-05-05 14:22:06,329 INFO MainThread:264993 [wandb_init.py:init():973] updated telemetry +2026-05-05 14:22:06,347 INFO MainThread:264993 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 14:22:07,470 INFO MainThread:264993 [wandb_init.py:init():1037] run resumed +2026-05-05 14:22:07,473 INFO MainThread:264993 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 14:22:07,919 INFO MainThread:264993 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 14:22:07,919 INFO MainThread:264993 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 14:22:07,919 INFO MainThread:264993 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 14:22:07,919 INFO MainThread:264993 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 14:22:07,930 INFO MainThread:264993 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 16:20:15,570 INFO wandb-AsyncioManager-main:264993 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 16:20:15,570 INFO wandb-AsyncioManager-main:264993 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc36b36d290aa3f7841aeabc51d39fab32afd264 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/config.yaml @@ -0,0 +1,390 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + myzwt5obildvp2c43p108x6swnfrmtr7: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py + - '-' + - trainer.resume=True + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_t769 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=900 + - trainer.save_ckpt_iter=100 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1176008605696" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769 + startedAt: "2026-05-04T19:32:30.219365Z" + writerId: myzwt5obildvp2c43p108x6swnfrmtr7 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 16 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "900" + offload_module_in_decoding: "False" + resume: "True" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/output.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..97bafe513f853139dec13a2c50e643b916b7b1f9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/output.log @@ -0,0 +1,83 @@ +[May 5, 2026 - 04:32:31 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 5, 2026 - 04:32:31 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 5, 2026 - 04:32:31 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 5, 2026 - 04:32:31 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 5, 2026 - 04:32:31 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 5, 2026 - 04:32:31 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 5, 2026 - 04:32:31 | INFO | fastgen.trainer:run:77 ] Starting training +[May 5, 2026 - 04:32:31 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 5, 2026 - 04:32:31 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 5, 2026 - 04:33:02 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 5, 2026 - 04:33:14 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 5, 2026 - 04:33:14 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 5, 2026 - 04:33:14 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 5, 2026 - 04:33:14 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 5, 2026 - 04:33:58 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 5, 2026 - 04:33:58 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 5, 2026 - 04:33:58 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 5, 2026 - 04:33:58 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 5, 2026 - 04:33:58 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 5, 2026 - 04:33:58 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 5, 2026 - 04:34:00 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:34:00 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 5, 2026 - 04:34:00 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 5, 2026 - 04:34:00 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 04:34:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:34:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.7s +[May 5, 2026 - 04:34:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 5, 2026 - 04:34:02 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 04:34:14 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:34:14 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.6s +[May 5, 2026 - 04:34:14 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 5, 2026 - 04:34:14 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 5, 2026 - 04:34:14 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 5, 2026 - 04:34:14 | INFO | fastgen.trainer:run:137 ] Loading checkpoints for resuming .. +[May 5, 2026 - 04:34:14 | INFO | fastgen.utils.checkpointer:load:436 ] Loading FSDP model from prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600 +[May 5, 2026 - 04:34:14 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key net... +[May 5, 2026 - 04:34:18 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key fake_score... +[May 5, 2026 - 04:34:19 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key net... +[May 5, 2026 - 04:34:19 | WARNING | fastgen.utils.checkpointer:load:526 ] Optimizer checkpoint for net missing at /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_optim; FASTGEN_ALLOW_MISSING_FSDP_OPTIM=1, so continuing with fresh optimizer state. +[May 5, 2026 - 04:34:19 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key fake_score... +[May 5, 2026 - 04:34:19 | WARNING | fastgen.utils.checkpointer:load:526 ] Optimizer checkpoint for fake_score missing at /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_optim; FASTGEN_ALLOW_MISSING_FSDP_OPTIM=1, so continuing with fresh optimizer state. +Traceback (most recent call last): + File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 46, in + main(config) + File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 37, in main + fastgen_trainer.run(model) + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 140, in run + iter_start = self.checkpointer.load( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/utils/checkpointer.py", line 561, in load + state = self._load_checkpoint(f"{path}.pth", device=device) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/utils/checkpointer.py", line 55, in _load_checkpoint + state = torch.load(path, map_location=device, weights_only=False) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/serialization.py", line 1573, in load + return _legacy_load( + ^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/serialization.py", line 1822, in _legacy_load + magic_number = pickle_module.load(f, **pickle_load_args) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +_pickle.UnpicklingError: invalid load key, '{'. +[rank0]: Traceback (most recent call last): +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 46, in +[rank0]: main(config) +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 37, in main +[rank0]: fastgen_trainer.run(model) +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 140, in run +[rank0]: iter_start = self.checkpointer.load( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/utils/checkpointer.py", line 561, in load +[rank0]: state = self._load_checkpoint(f"{path}.pth", device=device) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/utils/checkpointer.py", line 55, in _load_checkpoint +[rank0]: state = torch.load(path, map_location=device, weights_only=False) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/serialization.py", line 1573, in load +[rank0]: return _legacy_load( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/serialization.py", line 1822, in _legacy_load +[rank0]: magic_number = pickle_module.load(f, **pickle_load_args) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: _pickle.UnpicklingError: invalid load key, '{'. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..29750bf50a250ddc40731966c023a9573190d335 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-04T19:32:30.219365Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py", + "-", + "trainer.resume=True", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=900", + "trainer.save_ckpt_iter=100" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1176008605696" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "myzwt5obildvp2c43p108x6swnfrmtr7" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..88def40dcfe0d164c3cdc294a9b72c31928459cc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/files/wandb-summary.json @@ -0,0 +1 @@ +{"val0/sync_c_sample_5":6.618189811706543,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"val0/reconstructed":{"_type":"videos","captions":false,"count":10,"videos":[{"path":"media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028,"_type":"video-file"},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214","size":170922},{"size":173236,"_type":"video-file","path":"media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8"},{"size":249372,"_type":"video-file","path":"media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c"},{"path":"media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004,"_type":"video-file"},{"size":164630,"_type":"video-file","path":"media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395"},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4","sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490}]},"train/reward_weight_min":2.514177083969116,"train/reward_weight_r0":6556.8603515625,"profiler/optimizer_step_time":0.3926390949636698,"val0/sync_c_sample_8":5.9997477531433105,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"model/total_params":17150480992,"optimizer/iteration":600,"net/local_total_params":[355555624,355276072,355276072,355276072],"train/gan_loss_gen":0,"train/reward_sync_c_r3":2.8982486724853516,"fake_score/trainable_params":1421383840,"val0/sync_c_sample_4":4.935315132141113,"net/trainable_params":1421383840,"train/reward_sync_c_min":0.46097278594970703,"profiler/backward_pass_time":8.420644793659449,"train/reward_sync_c_r2":3.869823455810547,"val0/generated":{"count":10,"videos":[{"size":209988,"_type":"video-file","caption":"sync_c=6.320","path":"media/videos/val0/generated_500_ecf33b81c961fa87b44c.mp4","sha256":"ecf33b81c961fa87b44cec99381f95c1cde72a83f587d235c94a50f5aac66a23"},{"size":213943,"_type":"video-file","caption":"sync_c=5.968","path":"media/videos/val0/generated_500_6f432ba8a179d2cf9d42.mp4","sha256":"6f432ba8a179d2cf9d424d79bf07b4ade10a28b3c1cfd89e468d7f81dd090bab"},{"_type":"video-file","caption":"sync_c=4.882","path":"media/videos/val0/generated_500_bc93c864adf844e76807.mp4","sha256":"bc93c864adf844e7680719f2c5c40b0d77edadcb4d6fb7ca06e59fb3e7d2dad1","size":199544},{"sha256":"eff1a043cc25e64d562643c98c268fe4199cdcf622bcdbc1a707ab01e77da6e2","size":289550,"_type":"video-file","caption":"sync_c=5.328","path":"media/videos/val0/generated_500_eff1a043cc25e64d5626.mp4"},{"sha256":"268f148fe52acaf11e20aac1a1af12aa500c3136fd51aa19e8a5372e3b335ba0","size":253354,"_type":"video-file","caption":"sync_c=4.935","path":"media/videos/val0/generated_500_268f148fe52acaf11e20.mp4"},{"size":181831,"_type":"video-file","caption":"sync_c=6.618","path":"media/videos/val0/generated_500_37b567e82972db0905e6.mp4","sha256":"37b567e82972db0905e63ee297bd205aee77169d5955ca8aa7da16cc837e5d82"},{"_type":"video-file","caption":"sync_c=6.939","path":"media/videos/val0/generated_500_ad8196e3454985113065.mp4","sha256":"ad8196e3454985113065ef2de79c81e01214f8c3fc85ebcabe9fc759536ceed6","size":294877},{"_type":"video-file","caption":"sync_c=5.284","path":"media/videos/val0/generated_500_51967ebf199a3fc5b970.mp4","sha256":"51967ebf199a3fc5b970cbe5dac9566b2ae3e302a7290cf6327c1f6d32c8a8c1","size":195662},{"sha256":"5affe373089cb1c23cfd97e53a17ce81db46ece977d94e2188a8a312434eda8c","size":236735,"_type":"video-file","caption":"sync_c=6.000","path":"media/videos/val0/generated_500_5affe373089cb1c23cfd.mp4"},{"path":"media/videos/val0/generated_500_f437f159bf41584702e6.mp4","sha256":"f437f159bf41584702e6d6b26b5dd25d83f4534fa1f24ee9177ac348517df428","size":240818,"_type":"video-file","caption":"sync_c=3.343"}],"_type":"videos","captions":["sync_c=6.320","sync_c=5.968","sync_c=4.882","sync_c=5.328","sync_c=4.935","sync_c=6.618","sync_c=6.939","sync_c=5.284","sync_c=6.000","sync_c=3.343"]},"optimizer/lr_fake_score":2e-06,"val0/sync_c_mean":5.5616672992706295,"profiler/avg_forward_pass_time":74.30233238823712,"train/gan_loss_disc":0,"net/total_params":1421383840,"_timestamp":1.7775978670463598e+09,"train/reward_sync_c_r1":3.5555362701416016,"val0/total_loss":0,"train/fake_score_loss":0.0111083984375,"model/trainable_params":2842767680,"fake_score/total_params":1421383840,"train/reward_weight_r2":2297.660888671875,"train/reward_sync_c_max":6.674400806427002,"val0/sync_c_sample_7":5.2840046882629395,"val0/sync_c_sample_6":6.9387335777282715,"train/reward_weight_mean":60323.48828125,"val0/sync_c_sample_3":5.327848434448242,"train/total_loss":33094.310546875,"train/vsd_loss_weighted":33094.310546875,"_wandb":{"runtime":43279},"train_media/data/real":{"path":"media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4","sha256":"aebf0f3e2314a893fcf47216a0d6c68b02b30d80a7347c759681d9497a805c01","size":220015,"_type":"video-file"},"_step":600,"optimizer/non_finite_grads_count (model_key net)":0,"train_media/student/generation":{"_type":"video-file","path":"media/videos/train_media/student/generation_600_77974b3b30df0a6f7c8d.mp4","sha256":"77974b3b30df0a6f7c8d5a9372ae2a0707f086d0d97ca6d65aec402a77dbea3c","size":252862},"val0/sync_c_sample_2":4.882412910461426,"train/reward_sync_c_mean":4.160749912261963,"train/vsd_loss":0.5255126953125,"val0/sync_c_sample_1":5.9682159423828125,"train/reward_sync_c_r0":4.394133567810059,"model/local_trainable_params":[711111248,710552144,710552144,710552144],"optimizer/grad_norm (model_key net)":150645.515625,"train/reward_weight_max":627062.5625,"train/reward_weight_r1":1225.461181640625,"profiler/avg_iteration_time":165.81489205360413,"val0/sync_c_sample_0":6.319660186767578,"val0/sync_c_sample_9":3.3425445556640625,"train/vsd_loss_unweighted":0.5255126953125,"optimizer/lr_net":2e-06,"profiler/data_loading_time":0.0008898824453353882,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"_runtime":43279,"train/reward_weight_r3":329.1446533203125,"model/local_total_params":[4299192568,4297394424,4297394424,4297394424]} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..1508ed04c09dbd5cbc90ddce79c93141f1f17813 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-05T04:32:30.442019923+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpm0t5xe3n/port-3276270.txt","pid":3276270,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-05T04:32:30.44294658+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3276270} +{"time":"2026-05-05T04:32:30.442529909+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3276270-3289550-2332783787/socket","Net":"unix"}} +{"time":"2026-05-05T04:32:30.479900465+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-05T04:32:30.497285302+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T04:32:30.891894098+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T04:32:36.834677261+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"emmgphsbhz8f"} +{"time":"2026-05-05T04:34:19.598574957+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T04:34:19.598626626+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T04:34:19.59861836+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T04:34:19.598690244+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3276270-3289550-2332783787/socket","Net":"unix"}} +{"time":"2026-05-05T04:34:19.598722347+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T04:34:20.446798463+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-05T04:34:20.446815873+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-05T04:34:20.446823621+09:00","level":"INFO","msg":"server is closed"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5ad8a80274110abe7a6b6962cef32283bace06a4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T04:32:30.497628509+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T04:32:30.891781796+09:00","level":"INFO","msg":"stream: created new stream","id":"lvwg9pfc"} +{"time":"2026-05-05T04:32:30.891831918+09:00","level":"INFO","msg":"handler: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:32:30.891887972+09:00","level":"INFO","msg":"stream: started","id":"lvwg9pfc"} +{"time":"2026-05-05T04:32:30.891925853+09:00","level":"INFO","msg":"sender: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:32:30.891937487+09:00","level":"INFO","msg":"writer: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:34:19.598621707+09:00","level":"INFO","msg":"stream: closing","id":"lvwg9pfc"} +{"time":"2026-05-05T04:34:20.17106264+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T04:34:20.446535592+09:00","level":"INFO","msg":"handler: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:34:20.446613047+09:00","level":"INFO","msg":"sender: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:34:20.446619452+09:00","level":"INFO","msg":"stream: closed","id":"lvwg9pfc"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..26a2ba212253afab15c5dbe2f379d66a277b5d51 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug.log @@ -0,0 +1,22 @@ +2026-05-05 04:32:30,220 INFO MainThread:3276270 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 04:32:30,220 INFO MainThread:3276270 [wandb_setup.py:_flush():81] Configure stats pid to 3276270 +2026-05-05 04:32:30,220 INFO MainThread:3276270 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 04:32:30,220 INFO MainThread:3276270 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug.log +2026-05-05 04:32:30,220 INFO MainThread:3276270 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/logs/debug-internal.log +2026-05-05 04:32:30,220 INFO MainThread:3276270 [wandb_init.py:init():844] calling init triggers +2026-05-05 04:32:30,220 INFO MainThread:3276270 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '900', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 04:32:30,220 INFO MainThread:3276270 [wandb_init.py:init():892] starting backend +2026-05-05 04:32:30,480 INFO MainThread:3276270 [wandb_init.py:init():895] sending inform_init request +2026-05-05 04:32:30,494 INFO MainThread:3276270 [wandb_init.py:init():903] backend started and connected +2026-05-05 04:32:30,498 INFO MainThread:3276270 [wandb_init.py:init():973] updated telemetry +2026-05-05 04:32:30,520 INFO MainThread:3276270 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 04:32:31,471 INFO MainThread:3276270 [wandb_init.py:init():1037] run resumed +2026-05-05 04:32:31,474 INFO MainThread:3276270 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 04:32:31,828 INFO MainThread:3276270 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 04:32:31,828 INFO MainThread:3276270 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 04:32:31,828 INFO MainThread:3276270 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 04:32:31,829 INFO MainThread:3276270 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 04:32:31,836 INFO MainThread:3276270 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 04:34:19,598 INFO wandb-AsyncioManager-main:3276270 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 04:34:19,598 INFO wandb-AsyncioManager-main:3276270 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/run-lvwg9pfc.wandb b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/run-lvwg9pfc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f70aef9a0c66baf1c2f589e0bbe33ecada8f53bd Binary files /dev/null and b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_043230-lvwg9pfc/run-lvwg9pfc.wandb differ diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9fca96f222defc95997718968b9ce81604a4985 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/config.yaml @@ -0,0 +1,391 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + dptgdrwydeql7k92dx4qo02gt2stkxc5: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py + - '-' + - trainer.resume=True + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_t769 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=900 + - trainer.save_ckpt_iter=100 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1176008683520" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769 + startedAt: "2026-05-04T19:41:43.868670Z" + writerId: dptgdrwydeql7k92dx4qo02gt2stkxc5 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "900" + offload_module_in_decoding: "False" + resume: "True" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/output.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..fc49e11c1824ade61a371ea058ab7c356b24112d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/output.log @@ -0,0 +1,464 @@ +[May 5, 2026 - 04:41:45 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 5, 2026 - 04:41:45 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 5, 2026 - 04:41:45 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 5, 2026 - 04:41:45 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 5, 2026 - 04:41:45 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 5, 2026 - 04:41:45 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 5, 2026 - 04:41:45 | INFO | fastgen.trainer:run:77 ] Starting training +[May 5, 2026 - 04:41:45 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 5, 2026 - 04:41:45 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 5, 2026 - 04:42:21 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 5, 2026 - 04:42:32 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 5, 2026 - 04:42:32 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 5, 2026 - 04:42:32 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 5, 2026 - 04:42:32 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 5, 2026 - 04:43:16 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 5, 2026 - 04:43:16 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 5, 2026 - 04:43:16 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 5, 2026 - 04:43:16 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 5, 2026 - 04:43:16 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 5, 2026 - 04:43:16 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 5, 2026 - 04:43:18 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:43:18 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 5, 2026 - 04:43:18 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 5, 2026 - 04:43:18 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 04:43:20 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:43:20 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.0s +[May 5, 2026 - 04:43:20 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 5, 2026 - 04:43:20 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 04:43:32 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:43:32 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.1s +[May 5, 2026 - 04:43:32 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 5, 2026 - 04:43:32 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 5, 2026 - 04:43:32 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 5, 2026 - 04:43:32 | INFO | fastgen.trainer:run:137 ] Loading checkpoints for resuming .. +[May 5, 2026 - 04:43:32 | INFO | fastgen.utils.checkpointer:load:436 ] Loading FSDP model from prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600 +[May 5, 2026 - 04:43:32 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key net... +[May 5, 2026 - 04:43:34 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key fake_score... +[May 5, 2026 - 04:43:36 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key net... +[May 5, 2026 - 04:43:36 | WARNING | fastgen.utils.checkpointer:load:526 ] Optimizer checkpoint for net missing at /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_optim; FASTGEN_ALLOW_MISSING_FSDP_OPTIM=1, so continuing with fresh optimizer state. +[May 5, 2026 - 04:43:36 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key fake_score... +[May 5, 2026 - 04:43:36 | WARNING | fastgen.utils.checkpointer:load:526 ] Optimizer checkpoint for fake_score missing at /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_optim; FASTGEN_ALLOW_MISSING_FSDP_OPTIM=1, so continuing with fresh optimizer state. +[May 5, 2026 - 04:43:36 | INFO | fastgen.utils.checkpointer:load:564 ] Loading the scheduler_dict... +[May 5, 2026 - 04:43:36 | INFO | fastgen.utils.checkpointer:load:572 ] Loading the gradient scaler... +[May 5, 2026 - 04:43:36 | WARNING | fastgen.utils.checkpointer:load:577 ] Gradient scaler state is empty (likely saved from disabled scaler), skipping load. +[May 5, 2026 - 04:43:36 | INFO | fastgen.utils.checkpointer:load:580 ] Loading the callbacks... +[May 5, 2026 - 04:43:36 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 600. +[May 5, 2026 - 04:43:36 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17150.48 M total params (logical). +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4299.19 M total params LOCAL on rank 0. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 1. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 2. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 3. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 04:43:41 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 04:43:41 | INFO | fastgen.trainer:run:174 ] iter_start: 600 +Traceback (most recent call last): + File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 46, in + main(config) + File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 37, in main + fastgen_trainer.run(model) + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 194, in run + loss_map, outputs = self.train_step(model_ddp, model, data, iter_cur, grad_accum_iter) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 336, in train_step + model.optimizers_schedulers_step(iteration) + File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/methods/model.py", line 630, in optimizers_schedulers_step + optimizer.step() + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/lr_scheduler.py", line 166, in wrapper + return func.__get__(opt, opt.__class__)(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/optimizer.py", line 526, in wrapper + out = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/optimizer.py", line 81, in _use_grad + ret = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/adam.py", line 238, in step + has_complex = self._init_group( + ^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 1181, in _fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/adam.py", line 160, in _init_group + state = self.state[p] + ~~~~~~~~~~^^^ +KeyError: DTensor(local_tensor=tensor([[[[[-7.7994e-02, 5.3567e-02], + [-6.8809e-02, 4.1825e-03]]], + + + [[[ 3.5393e-02, -5.3090e-02], + [ 3.9058e-02, 2.2043e-02]]], + + + [[[ 1.8552e-02, -6.5991e-02], + [ 4.9795e-02, -4.4065e-02]]], + + + ..., + + + [[[ 7.9077e-04, 4.4253e-04], + [-1.2932e-04, 4.0694e-05]]], + + + [[[-3.1836e-04, 2.7853e-04], + [ 4.0060e-04, 1.0321e-04]]], + + + [[[-1.5851e-03, -9.3719e-04], + [ 1.6040e-04, 1.0815e-04]]]], + + + + [[[[ 4.3391e-03, 4.2336e-02], + [-2.9585e-02, 1.8308e-02]]], + + + [[[-6.6034e-02, -2.8392e-02], + [-2.5318e-02, 6.4808e-02]]], + + + [[[-3.4435e-02, -2.1360e-02], + [ 4.1628e-02, 4.3757e-02]]], + + + ..., + + + [[[ 7.7770e-04, -3.4096e-05], + [ 7.3310e-04, 2.6909e-04]]], + + + [[[-4.5127e-05, 5.7697e-05], + [ 3.0869e-04, -1.3565e-05]]], + + + [[[ 1.1159e-03, 2.1580e-03], + [ 1.5331e-03, 2.0081e-03]]]], + + + + [[[[ 1.3987e-03, -1.9610e-02], + [-4.7619e-04, -4.0242e-03]]], + + + [[[-1.0187e-02, 1.0558e-02], + [-2.5391e-05, -3.4531e-02]]], + + + [[[ 3.8815e-03, -4.1860e-02], + [-2.0291e-02, -3.2869e-03]]], + + + ..., + + + [[[-1.8654e-03, -1.8156e-03], + [-1.7963e-03, -2.2436e-03]]], + + + [[[ 1.1354e-03, 5.3782e-04], + [ 5.5527e-04, 4.6145e-04]]], + + + [[[ 1.2318e-03, 1.1071e-03], + [ 8.1610e-04, 1.1118e-03]]]], + + + + ..., + + + + [[[[-1.9200e-03, 6.3843e-02], + [-4.6839e-03, 6.4843e-02]]], + + + [[[ 1.7394e-02, -4.7029e-02], + [ 2.9194e-02, 9.6880e-02]]], + + + [[[-3.9967e-02, -4.3392e-02], + [-2.0641e-02, 6.3817e-02]]], + + + ..., + + + [[[ 1.1166e-04, -1.0180e-04], + [-7.6623e-05, -1.3208e-04]]], + + + [[[-5.1262e-04, -1.3557e-03], + [ 1.2466e-03, 5.6050e-04]]], + + + [[[ 7.7446e-05, -9.6470e-04], + [ 1.6875e-03, 1.2021e-03]]]], + + + + [[[[ 1.1064e-02, 8.6103e-02], + [ 6.6549e-02, 5.9082e-02]]], + + + [[[-1.7891e-02, 1.2033e-01], + [ 1.2202e-02, 1.2779e-01]]], + + + [[[-3.3115e-02, -1.3636e-01], + [-8.5148e-02, -6.0225e-02]]], + + + ..., + + + [[[-8.8611e-04, -8.6880e-04], + [-7.6463e-04, -6.1143e-04]]], + + + [[[-3.3544e-04, -4.7982e-04], + [ 4.1290e-05, -1.8030e-04]]], + + + [[[ 2.7208e-04, 5.1157e-04], + [ 5.0452e-04, 8.0516e-04]]]], + + + + [[[[-1.7144e-01, -8.9599e-02], + [-1.6007e-01, -1.0046e-01]]], + + + [[[-6.2192e-02, 1.0511e-01], + [ 1.7952e-01, 1.0169e-02]]], + + + [[[-8.9964e-02, 1.9622e-01], + [ 1.6797e-02, 6.5938e-02]]], + + + ..., + + + [[[ 9.5227e-06, 9.4348e-05], + [-5.9497e-04, -1.1351e-03]]], + + + [[[ 2.3090e-04, 8.4907e-05], + [-1.8821e-04, -2.2280e-04]]], + + + [[[ 4.0714e-04, 2.2092e-04], + [ 1.1834e-04, 4.6117e-04]]]]], device='cuda:0'), device_mesh=DeviceMesh((4,), 'cuda', stride=(1,)), placements=(Shard(dim=0),)) +[rank0]: Traceback (most recent call last): +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 46, in +[rank0]: main(config) +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/train.py", line 37, in main +[rank0]: fastgen_trainer.run(model) +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 194, in run +[rank0]: loss_map, outputs = self.train_step(model_ddp, model, data, iter_cur, grad_accum_iter) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/trainer.py", line 336, in train_step +[rank0]: model.optimizers_schedulers_step(iteration) +[rank0]: File "/home/work/.local/hyunbin/FastGen-redmd/fastgen/methods/model.py", line 630, in optimizers_schedulers_step +[rank0]: optimizer.step() +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/lr_scheduler.py", line 166, in wrapper +[rank0]: return func.__get__(opt, opt.__class__)(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/optimizer.py", line 526, in wrapper +[rank0]: out = func(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/optimizer.py", line 81, in _use_grad +[rank0]: ret = func(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/adam.py", line 238, in step +[rank0]: has_complex = self._init_group( +[rank0]: ^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 1181, in _fn +[rank0]: return fn(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torch/optim/adam.py", line 160, in _init_group +[rank0]: state = self.state[p] +[rank0]: ~~~~~~~~~~^^^ +[rank0]: KeyError: DTensor(local_tensor=tensor([[[[[-7.7994e-02, 5.3567e-02], +[rank0]: [-6.8809e-02, 4.1825e-03]]], + + +[rank0]: [[[ 3.5393e-02, -5.3090e-02], +[rank0]: [ 3.9058e-02, 2.2043e-02]]], + + +[rank0]: [[[ 1.8552e-02, -6.5991e-02], +[rank0]: [ 4.9795e-02, -4.4065e-02]]], + + +[rank0]: ..., + + +[rank0]: [[[ 7.9077e-04, 4.4253e-04], +[rank0]: [-1.2932e-04, 4.0694e-05]]], + + +[rank0]: [[[-3.1836e-04, 2.7853e-04], +[rank0]: [ 4.0060e-04, 1.0321e-04]]], + + +[rank0]: [[[-1.5851e-03, -9.3719e-04], +[rank0]: [ 1.6040e-04, 1.0815e-04]]]], + + + +[rank0]: [[[[ 4.3391e-03, 4.2336e-02], +[rank0]: [-2.9585e-02, 1.8308e-02]]], + + +[rank0]: [[[-6.6034e-02, -2.8392e-02], +[rank0]: [-2.5318e-02, 6.4808e-02]]], + + +[rank0]: [[[-3.4435e-02, -2.1360e-02], +[rank0]: [ 4.1628e-02, 4.3757e-02]]], + + +[rank0]: ..., + + +[rank0]: [[[ 7.7770e-04, -3.4096e-05], +[rank0]: [ 7.3310e-04, 2.6909e-04]]], + + +[rank0]: [[[-4.5127e-05, 5.7697e-05], +[rank0]: [ 3.0869e-04, -1.3565e-05]]], + + +[rank0]: [[[ 1.1159e-03, 2.1580e-03], +[rank0]: [ 1.5331e-03, 2.0081e-03]]]], + + + +[rank0]: [[[[ 1.3987e-03, -1.9610e-02], +[rank0]: [-4.7619e-04, -4.0242e-03]]], + + +[rank0]: [[[-1.0187e-02, 1.0558e-02], +[rank0]: [-2.5391e-05, -3.4531e-02]]], + + +[rank0]: [[[ 3.8815e-03, -4.1860e-02], +[rank0]: [-2.0291e-02, -3.2869e-03]]], + + +[rank0]: ..., + + +[rank0]: [[[-1.8654e-03, -1.8156e-03], +[rank0]: [-1.7963e-03, -2.2436e-03]]], + + +[rank0]: [[[ 1.1354e-03, 5.3782e-04], +[rank0]: [ 5.5527e-04, 4.6145e-04]]], + + +[rank0]: [[[ 1.2318e-03, 1.1071e-03], +[rank0]: [ 8.1610e-04, 1.1118e-03]]]], + + + +[rank0]: ..., + + + +[rank0]: [[[[-1.9200e-03, 6.3843e-02], +[rank0]: [-4.6839e-03, 6.4843e-02]]], + + +[rank0]: [[[ 1.7394e-02, -4.7029e-02], +[rank0]: [ 2.9194e-02, 9.6880e-02]]], + + +[rank0]: [[[-3.9967e-02, -4.3392e-02], +[rank0]: [-2.0641e-02, 6.3817e-02]]], + + +[rank0]: ..., + + +[rank0]: [[[ 1.1166e-04, -1.0180e-04], +[rank0]: [-7.6623e-05, -1.3208e-04]]], + + +[rank0]: [[[-5.1262e-04, -1.3557e-03], +[rank0]: [ 1.2466e-03, 5.6050e-04]]], + + +[rank0]: [[[ 7.7446e-05, -9.6470e-04], +[rank0]: [ 1.6875e-03, 1.2021e-03]]]], + + + +[rank0]: [[[[ 1.1064e-02, 8.6103e-02], +[rank0]: [ 6.6549e-02, 5.9082e-02]]], + + +[rank0]: [[[-1.7891e-02, 1.2033e-01], +[rank0]: [ 1.2202e-02, 1.2779e-01]]], + + +[rank0]: [[[-3.3115e-02, -1.3636e-01], +[rank0]: [-8.5148e-02, -6.0225e-02]]], + + +[rank0]: ..., + + +[rank0]: [[[-8.8611e-04, -8.6880e-04], +[rank0]: [-7.6463e-04, -6.1143e-04]]], + + +[rank0]: [[[-3.3544e-04, -4.7982e-04], +[rank0]: [ 4.1290e-05, -1.8030e-04]]], + + +[rank0]: [[[ 2.7208e-04, 5.1157e-04], +[rank0]: [ 5.0452e-04, 8.0516e-04]]]], + + + +[rank0]: [[[[-1.7144e-01, -8.9599e-02], +[rank0]: [-1.6007e-01, -1.0046e-01]]], + + +[rank0]: [[[-6.2192e-02, 1.0511e-01], +[rank0]: [ 1.7952e-01, 1.0169e-02]]], + + +[rank0]: [[[-8.9964e-02, 1.9622e-01], +[rank0]: [ 1.6797e-02, 6.5938e-02]]], + + +[rank0]: ..., + + +[rank0]: [[[ 9.5227e-06, 9.4348e-05], +[rank0]: [-5.9497e-04, -1.1351e-03]]], + + +[rank0]: [[[ 2.3090e-04, 8.4907e-05], +[rank0]: [-1.8821e-04, -2.2280e-04]]], + + +[rank0]: [[[ 4.0714e-04, 2.2092e-04], +[rank0]: [ 1.1834e-04, 4.6117e-04]]]]], device='cuda:0'), device_mesh=DeviceMesh((4,), 'cuda', stride=(1,)), placements=(Shard(dim=0),)) diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..da47f441ba97b8f32c0631c2a0be29cb8832a67b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-04T19:41:43.868670Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py", + "-", + "trainer.resume=True", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=900", + "trainer.save_ckpt_iter=100" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1176008683520" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "dptgdrwydeql7k92dx4qo02gt2stkxc5" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f4a9b3cf5d6dac87d04f320731cc4e158322924b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/files/wandb-summary.json @@ -0,0 +1 @@ +{"optimizer/non_finite_grads_count (model_key net)":0,"train_media/student/generation":{"_type":"video-file","path":"media/videos/train_media/student/generation_600_77974b3b30df0a6f7c8d.mp4","sha256":"77974b3b30df0a6f7c8d5a9372ae2a0707f086d0d97ca6d65aec402a77dbea3c","size":252862},"val0/total_loss":0,"profiler/optimizer_step_time":0.3926390949636698,"val0/reconstructed":{"_type":"videos","captions":false,"count":10,"videos":[{"path":"media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028,"_type":"video-file"},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214","size":170922},{"path":"media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8","size":173236,"_type":"video-file"},{"path":"media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372,"_type":"video-file"},{"path":"media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004,"_type":"video-file"},{"sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630,"_type":"video-file","path":"media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4"},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4","sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001},{"path":"media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490,"_type":"video-file"}]},"fake_score/trainable_params":1421383840,"_runtime":43441.495016429,"val0/sync_c_sample_8":5.9997477531433105,"train/reward_sync_c_r3":2.8982486724853516,"optimizer/grad_norm (model_key net)":0,"val0/sync_c_sample_7":5.2840046882629395,"train/vsd_loss":0.5255126953125,"profiler/data_loading_time":0.0008898824453353882,"val0/sync_c_mean":5.5616672992706295,"val0/sync_c_sample_5":6.618189811706543,"train/vsd_loss_weighted":33094.310546875,"train/reward_weight_r0":6556.8603515625,"train/reward_weight_r3":329.1446533203125,"train/reward_sync_c_max":6.674400806427002,"profiler/backward_pass_time":8.420644793659449,"net/total_params":1421383840,"optimizer/iteration":600,"fake_score/total_params":1421383840,"val0/sync_c_sample_6":6.9387335777282715,"_wandb":{"runtime":43441},"train/reward_weight_max":627062.5625,"train/reward_sync_c_r0":4.394133567810059,"val0/sync_c_sample_9":3.3425445556640625,"model/local_total_params":[4299192568,4297394424,4297394424,4297394424],"train/reward_sync_c_mean":4.160749912261963,"train/reward_weight_min":2.514177083969116,"train_media/data/real":{"_type":"video-file","path":"media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4","sha256":"aebf0f3e2314a893fcf47216a0d6c68b02b30d80a7347c759681d9497a805c01","size":220015},"val0/sync_c_sample_3":5.327848434448242,"train/fake_score_loss":0.0111083984375,"train/gan_loss_gen":0,"_step":601,"optimizer/lr_fake_score":2e-06,"val0/sync_c_sample_0":6.319660186767578,"profiler/avg_iteration_time":165.81489205360413,"train/reward_sync_c_min":0.46097278594970703,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"model/local_trainable_params":[711111248,710552144,710552144,710552144],"train/total_loss":33094.310546875,"train/vsd_loss_unweighted":0.5255126953125,"train/reward_sync_c_r1":3.5555362701416016,"model/trainable_params":2842767680,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"val0/sync_c_sample_2":4.882412910461426,"_timestamp":1.7779238660131726e+09,"train/gan_loss_disc":0,"train/reward_weight_mean":60323.48828125,"profiler/avg_forward_pass_time":74.30233238823712,"net/local_total_params":[355555624,355276072,355276072,355276072],"train/reward_sync_c_r2":3.869823455810547,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"net/trainable_params":1421383840,"train/reward_weight_r1":1225.461181640625,"model/total_params":17150480992,"train/reward_weight_r2":2297.660888671875,"optimizer/lr_net":2e-06,"val0/generated":{"count":10,"videos":[{"_type":"video-file","caption":"sync_c=6.320","path":"media/videos/val0/generated_500_ecf33b81c961fa87b44c.mp4","sha256":"ecf33b81c961fa87b44cec99381f95c1cde72a83f587d235c94a50f5aac66a23","size":209988},{"path":"media/videos/val0/generated_500_6f432ba8a179d2cf9d42.mp4","sha256":"6f432ba8a179d2cf9d424d79bf07b4ade10a28b3c1cfd89e468d7f81dd090bab","size":213943,"_type":"video-file","caption":"sync_c=5.968"},{"caption":"sync_c=4.882","path":"media/videos/val0/generated_500_bc93c864adf844e76807.mp4","sha256":"bc93c864adf844e7680719f2c5c40b0d77edadcb4d6fb7ca06e59fb3e7d2dad1","size":199544,"_type":"video-file"},{"_type":"video-file","caption":"sync_c=5.328","path":"media/videos/val0/generated_500_eff1a043cc25e64d5626.mp4","sha256":"eff1a043cc25e64d562643c98c268fe4199cdcf622bcdbc1a707ab01e77da6e2","size":289550},{"_type":"video-file","caption":"sync_c=4.935","path":"media/videos/val0/generated_500_268f148fe52acaf11e20.mp4","sha256":"268f148fe52acaf11e20aac1a1af12aa500c3136fd51aa19e8a5372e3b335ba0","size":253354},{"size":181831,"_type":"video-file","caption":"sync_c=6.618","path":"media/videos/val0/generated_500_37b567e82972db0905e6.mp4","sha256":"37b567e82972db0905e63ee297bd205aee77169d5955ca8aa7da16cc837e5d82"},{"_type":"video-file","caption":"sync_c=6.939","path":"media/videos/val0/generated_500_ad8196e3454985113065.mp4","sha256":"ad8196e3454985113065ef2de79c81e01214f8c3fc85ebcabe9fc759536ceed6","size":294877},{"size":195662,"_type":"video-file","caption":"sync_c=5.284","path":"media/videos/val0/generated_500_51967ebf199a3fc5b970.mp4","sha256":"51967ebf199a3fc5b970cbe5dac9566b2ae3e302a7290cf6327c1f6d32c8a8c1"},{"sha256":"5affe373089cb1c23cfd97e53a17ce81db46ece977d94e2188a8a312434eda8c","size":236735,"_type":"video-file","caption":"sync_c=6.000","path":"media/videos/val0/generated_500_5affe373089cb1c23cfd.mp4"},{"_type":"video-file","caption":"sync_c=3.343","path":"media/videos/val0/generated_500_f437f159bf41584702e6.mp4","sha256":"f437f159bf41584702e6d6b26b5dd25d83f4534fa1f24ee9177ac348517df428","size":240818}],"_type":"videos","captions":["sync_c=6.320","sync_c=5.968","sync_c=4.882","sync_c=5.328","sync_c=4.935","sync_c=6.618","sync_c=6.939","sync_c=5.284","sync_c=6.000","sync_c=3.343"]},"val0/sync_c_sample_4":4.935315132141113,"val0/sync_c_sample_1":5.9682159423828125} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b717ea6f7d5e7b8ab1440c2c286fa915ca7d3781 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-05T04:41:43.95824347+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprppp8cp7/port-3314831.txt","pid":3314831,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-05T04:41:43.958726266+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3314831} +{"time":"2026-05-05T04:41:43.958720596+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3314831-3328527-1493474242/socket","Net":"unix"}} +{"time":"2026-05-05T04:41:44.137115865+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-05T04:41:44.153416214+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T04:41:44.524269447+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T04:41:50.390210036+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"9ioq2qvossaj"} +{"time":"2026-05-05T04:44:27.541531575+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T04:44:27.54158111+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T04:44:27.541573455+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T04:44:27.541657774+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3314831-3328527-1493474242/socket","Net":"unix"}} +{"time":"2026-05-05T04:44:27.541677409+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T04:44:29.459119981+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-05T04:44:29.459137199+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-05T04:44:29.459145516+09:00","level":"INFO","msg":"server is closed"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5f534a9aee9e81f4e56df3e9460155468f8460f8 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T04:41:44.15349848+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T04:41:44.524128231+09:00","level":"INFO","msg":"stream: created new stream","id":"lvwg9pfc"} +{"time":"2026-05-05T04:41:44.524178627+09:00","level":"INFO","msg":"handler: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:41:44.524263176+09:00","level":"INFO","msg":"stream: started","id":"lvwg9pfc"} +{"time":"2026-05-05T04:41:44.524285589+09:00","level":"INFO","msg":"sender: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:41:44.524288675+09:00","level":"INFO","msg":"writer: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:44:27.541584576+09:00","level":"INFO","msg":"stream: closing","id":"lvwg9pfc"} +{"time":"2026-05-05T04:44:29.072428572+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T04:44:29.458855395+09:00","level":"INFO","msg":"handler: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:44:29.458921066+09:00","level":"INFO","msg":"sender: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:44:29.458939275+09:00","level":"INFO","msg":"stream: closed","id":"lvwg9pfc"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2e3a11ad13cc1f4f11dd80a91c27772117af9fb5 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug.log @@ -0,0 +1,22 @@ +2026-05-05 04:41:43,869 INFO MainThread:3314831 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 04:41:43,869 INFO MainThread:3314831 [wandb_setup.py:_flush():81] Configure stats pid to 3314831 +2026-05-05 04:41:43,869 INFO MainThread:3314831 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 04:41:43,869 INFO MainThread:3314831 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug.log +2026-05-05 04:41:43,870 INFO MainThread:3314831 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/logs/debug-internal.log +2026-05-05 04:41:43,870 INFO MainThread:3314831 [wandb_init.py:init():844] calling init triggers +2026-05-05 04:41:43,870 INFO MainThread:3314831 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '900', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 04:41:43,870 INFO MainThread:3314831 [wandb_init.py:init():892] starting backend +2026-05-05 04:41:44,137 INFO MainThread:3314831 [wandb_init.py:init():895] sending inform_init request +2026-05-05 04:41:44,151 INFO MainThread:3314831 [wandb_init.py:init():903] backend started and connected +2026-05-05 04:41:44,155 INFO MainThread:3314831 [wandb_init.py:init():973] updated telemetry +2026-05-05 04:41:44,180 INFO MainThread:3314831 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 04:41:45,042 INFO MainThread:3314831 [wandb_init.py:init():1037] run resumed +2026-05-05 04:41:45,045 INFO MainThread:3314831 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 04:41:45,387 INFO MainThread:3314831 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 04:41:45,387 INFO MainThread:3314831 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 04:41:45,387 INFO MainThread:3314831 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 04:41:45,387 INFO MainThread:3314831 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 04:41:45,393 INFO MainThread:3314831 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 04:44:27,541 INFO wandb-AsyncioManager-main:3314831 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 04:44:27,541 INFO wandb-AsyncioManager-main:3314831 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/run-lvwg9pfc.wandb b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/run-lvwg9pfc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4db989ef95482b44a59d6bd4c96e7acdc5cef6bc Binary files /dev/null and b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044143-lvwg9pfc/run-lvwg9pfc.wandb differ diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca6e6bbedb8200c9d1631667fa96efd2a6f6193c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/config.yaml @@ -0,0 +1,391 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + vbp01sd3cntkhlc4428phadvyen76ql4: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py + - '-' + - trainer.resume=True + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_t769 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=900 + - trainer.save_ckpt_iter=100 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1176008798208" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769 + startedAt: "2026-05-04T19:49:12.585111Z" + writerId: vbp01sd3cntkhlc4428phadvyen76ql4 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "900" + offload_module_in_decoding: "False" + resume: "True" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/output.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ff217b8568ca3d7bcb4359d2ec9233d38e88034d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/output.log @@ -0,0 +1,114 @@ +[May 5, 2026 - 04:49:14 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 5, 2026 - 04:49:14 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 5, 2026 - 04:49:14 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 5, 2026 - 04:49:14 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 5, 2026 - 04:49:14 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 5, 2026 - 04:49:14 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 5, 2026 - 04:49:14 | INFO | fastgen.trainer:run:77 ] Starting training +[May 5, 2026 - 04:49:14 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 5, 2026 - 04:49:14 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 5, 2026 - 04:49:18 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 5, 2026 - 04:49:30 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 5, 2026 - 04:49:30 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 5, 2026 - 04:49:30 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 5, 2026 - 04:49:30 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 5, 2026 - 04:50:13 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 5, 2026 - 04:50:13 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 5, 2026 - 04:50:13 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 5, 2026 - 04:50:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 5, 2026 - 04:50:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 5, 2026 - 04:50:13 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 5, 2026 - 04:50:15 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:50:15 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 5, 2026 - 04:50:15 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 5, 2026 - 04:50:15 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 04:50:17 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:50:17 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.7s +[May 5, 2026 - 04:50:17 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 5, 2026 - 04:50:17 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 04:50:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 04:50:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.1s +[May 5, 2026 - 04:50:29 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 5, 2026 - 04:50:29 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 5, 2026 - 04:50:29 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 5, 2026 - 04:50:29 | INFO | fastgen.trainer:run:137 ] Loading checkpoints for resuming .. +[May 5, 2026 - 04:50:29 | INFO | fastgen.utils.checkpointer:load:436 ] Loading FSDP model from prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600 +[May 5, 2026 - 04:50:29 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key net... +[May 5, 2026 - 04:50:31 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key fake_score... +[May 5, 2026 - 04:50:32 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key net... +[May 5, 2026 - 04:50:32 | WARNING | fastgen.utils.checkpointer:load:526 ] Optimizer checkpoint for net missing at /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_optim; FASTGEN_ALLOW_MISSING_FSDP_OPTIM=1, so continuing with fresh optimizer state. +[May 5, 2026 - 04:50:32 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key fake_score... +[May 5, 2026 - 04:50:32 | WARNING | fastgen.utils.checkpointer:load:526 ] Optimizer checkpoint for fake_score missing at /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_optim; FASTGEN_ALLOW_MISSING_FSDP_OPTIM=1, so continuing with fresh optimizer state. +[May 5, 2026 - 04:50:32 | INFO | fastgen.utils.checkpointer:load:565 ] Loading the scheduler_dict... +[May 5, 2026 - 04:50:32 | INFO | fastgen.utils.checkpointer:load:573 ] Loading the gradient scaler... +[May 5, 2026 - 04:50:32 | WARNING | fastgen.utils.checkpointer:load:578 ] Gradient scaler state is empty (likely saved from disabled scaler), skipping load. +[May 5, 2026 - 04:50:32 | INFO | fastgen.utils.checkpointer:load:581 ] Loading the callbacks... +[May 5, 2026 - 04:50:32 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 600. +[May 5, 2026 - 04:50:32 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17150.48 M total params (logical). +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4299.19 M total params LOCAL on rank 0. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 1. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 2. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 3. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 04:50:38 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 04:50:38 | INFO | fastgen.trainer:run:174 ] iter_start: 600 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.408033 37.454948 37.356743 +peak_gpu_mem_gb 58.406074 58.407636 58.405553 +peak_gpu_mem_reserved_gb 60.617676 60.619141 60.617188 +util 78.000000 82.000000 75.000000 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 601 : data loading time 0.98 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 601 : avg forward pass time 10.23 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 601 : backward pass time 10.11 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 601 : optimizer step time 1.74 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 601-------------------- +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:51:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 601 that is less than the current step 602. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.583454 37.630123 37.532032 +peak_gpu_mem_gb 61.054405 61.056748 61.053624 +peak_gpu_mem_reserved_gb 62.397461 62.400391 62.396484 +util 97.750000 99.000000 96.000000 +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 602 : avg iteration time 43.45 seconds +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 602 : data loading time 0.00 +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 602 : avg forward pass time 10.76 +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 602 : backward pass time 10.17 +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 602 : optimizer step time 1.59 +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 602-------------------- +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:52:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.583653 37.630230 37.532108 +peak_gpu_mem_gb 61.054405 61.056748 61.053624 +peak_gpu_mem_reserved_gb 62.377930 62.380859 62.376953 +util 96.750000 100.000000 89.000000 +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 603 : avg iteration time 48.64 seconds +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 603 : data loading time 0.00 +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 603 : avg forward pass time 13.35 +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 603 : backward pass time 10.18 +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 603 : optimizer step time 1.59 +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 603-------------------- +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:52:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ed332fb35cb2c3c1f7c65c2c58026e0e2f10068f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-04T19:49:12.585111Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py", + "-", + "trainer.resume=True", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=900", + "trainer.save_ckpt_iter=100" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1176008798208" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "vbp01sd3cntkhlc4428phadvyen76ql4" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d547cb94695cd29e04c687df62f349c940afb521 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/reward_weight_r0":6556.8603515625,"train/vsd_loss_weighted":33094.310546875,"val0/reconstructed":{"videos":[{"sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028,"_type":"video-file","path":"media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4"},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214","size":170922},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8","size":173236},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004},{"size":164630,"_type":"video-file","path":"media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395"},{"path":"media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986,"_type":"video-file"},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516},{"size":214001,"_type":"video-file","path":"media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4","sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac"},{"_type":"video-file","path":"media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490}],"_type":"videos","captions":false,"count":10},"_step":603,"val0/total_loss":0,"val0/sync_c_sample_2":4.882412910461426,"train/total_loss":0.01190185546875,"val0/sync_c_sample_3":5.327848434448242,"train_media/student/generation":{"_type":"video-file","path":"media/videos/train_media/student/generation_600_77974b3b30df0a6f7c8d.mp4","sha256":"77974b3b30df0a6f7c8d5a9372ae2a0707f086d0d97ca6d65aec402a77dbea3c","size":252862},"optimizer/iteration":603,"train/reward_weight_r2":2297.660888671875,"optimizer/grad_norm (model_key net)":0,"optimizer/lr_fake_score":2e-06,"train/gan_loss_disc":0,"fake_score/trainable_params":1421383840,"model/trainable_params":2842767680,"profiler/avg_forward_pass_time":13.352451990358531,"train/gan_loss_gen":0,"train/reward_sync_c_max":6.674400806427002,"_timestamp":1.7779243760351436e+09,"val0/generated":{"_type":"videos","captions":["sync_c=6.320","sync_c=5.968","sync_c=4.882","sync_c=5.328","sync_c=4.935","sync_c=6.618","sync_c=6.939","sync_c=5.284","sync_c=6.000","sync_c=3.343"],"count":10,"videos":[{"caption":"sync_c=6.320","path":"media/videos/val0/generated_500_ecf33b81c961fa87b44c.mp4","sha256":"ecf33b81c961fa87b44cec99381f95c1cde72a83f587d235c94a50f5aac66a23","size":209988,"_type":"video-file"},{"sha256":"6f432ba8a179d2cf9d424d79bf07b4ade10a28b3c1cfd89e468d7f81dd090bab","size":213943,"_type":"video-file","caption":"sync_c=5.968","path":"media/videos/val0/generated_500_6f432ba8a179d2cf9d42.mp4"},{"caption":"sync_c=4.882","path":"media/videos/val0/generated_500_bc93c864adf844e76807.mp4","sha256":"bc93c864adf844e7680719f2c5c40b0d77edadcb4d6fb7ca06e59fb3e7d2dad1","size":199544,"_type":"video-file"},{"caption":"sync_c=5.328","path":"media/videos/val0/generated_500_eff1a043cc25e64d5626.mp4","sha256":"eff1a043cc25e64d562643c98c268fe4199cdcf622bcdbc1a707ab01e77da6e2","size":289550,"_type":"video-file"},{"sha256":"268f148fe52acaf11e20aac1a1af12aa500c3136fd51aa19e8a5372e3b335ba0","size":253354,"_type":"video-file","caption":"sync_c=4.935","path":"media/videos/val0/generated_500_268f148fe52acaf11e20.mp4"},{"caption":"sync_c=6.618","path":"media/videos/val0/generated_500_37b567e82972db0905e6.mp4","sha256":"37b567e82972db0905e63ee297bd205aee77169d5955ca8aa7da16cc837e5d82","size":181831,"_type":"video-file"},{"caption":"sync_c=6.939","path":"media/videos/val0/generated_500_ad8196e3454985113065.mp4","sha256":"ad8196e3454985113065ef2de79c81e01214f8c3fc85ebcabe9fc759536ceed6","size":294877,"_type":"video-file"},{"caption":"sync_c=5.284","path":"media/videos/val0/generated_500_51967ebf199a3fc5b970.mp4","sha256":"51967ebf199a3fc5b970cbe5dac9566b2ae3e302a7290cf6327c1f6d32c8a8c1","size":195662,"_type":"video-file"},{"size":236735,"_type":"video-file","caption":"sync_c=6.000","path":"media/videos/val0/generated_500_5affe373089cb1c23cfd.mp4","sha256":"5affe373089cb1c23cfd97e53a17ce81db46ece977d94e2188a8a312434eda8c"},{"_type":"video-file","caption":"sync_c=3.343","path":"media/videos/val0/generated_500_f437f159bf41584702e6.mp4","sha256":"f437f159bf41584702e6d6b26b5dd25d83f4534fa1f24ee9177ac348517df428","size":240818}]},"train/reward_weight_r3":329.1446533203125,"net/local_total_params":[355555624,355276072,355276072,355276072],"train/reward_sync_c_r3":2.8982486724853516,"train/reward_weight_mean":60323.48828125,"net/trainable_params":1421383840,"train/reward_sync_c_r2":3.869823455810547,"train/reward_sync_c_mean":4.160749912261963,"train_media/data/real":{"_type":"video-file","path":"media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4","sha256":"aebf0f3e2314a893fcf47216a0d6c68b02b30d80a7347c759681d9497a805c01","size":220015},"train/reward_weight_max":627062.5625,"train/reward_weight_r1":1225.461181640625,"profiler/backward_pass_time":10.176258036866784,"val0/sync_c_sample_0":6.319660186767578,"val0/sync_c_sample_7":5.2840046882629395,"model/total_params":17150480992,"train/reward_sync_c_r0":4.394133567810059,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"_runtime":43705.001881908,"val0/sync_c_sample_1":5.9682159423828125,"val0/sync_c_sample_8":5.9997477531433105,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"train/fake_score_loss":0.01190185546875,"optimizer/lr_net":2e-06,"train/reward_weight_min":2.514177083969116,"profiler/data_loading_time":0.0008804351091384888,"profiler/optimizer_step_time":1.5870651416480541,"model/local_trainable_params":[711111248,710552144,710552144,710552144],"val0/sync_c_sample_5":6.618189811706543,"fake_score/total_params":1421383840,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"val0/sync_c_sample_4":4.935315132141113,"train/vsd_loss":0.5255126953125,"profiler/avg_iteration_time":48.63590955734253,"_wandb":{"runtime":43705},"val0/sync_c_sample_6":6.9387335777282715,"net/total_params":1421383840,"train/vsd_loss_unweighted":0.5255126953125,"model/local_total_params":[4299192568,4297394424,4297394424,4297394424],"optimizer/non_finite_grads_count (model_key net)":0,"val0/sync_c_sample_9":3.3425445556640625,"train/reward_sync_c_r1":3.5555362701416016,"train/reward_sync_c_min":0.46097278594970703,"val0/sync_c_mean":5.5616672992706295} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b1853ac97c6c743b34bf5de117078c1df92fb855 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2026-05-05T04:49:12.637758849+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpcwu2okh2/port-3344108.txt","pid":3344108,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-05T04:49:12.63823768+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3344108} +{"time":"2026-05-05T04:49:12.638233447+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3344108-3356955-2788809641/socket","Net":"unix"}} +{"time":"2026-05-05T04:49:12.822111871+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-05T04:49:12.836628997+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T04:49:13.227416291+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T04:49:19.125122446+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8lvp2qshqt5o"} +{"time":"2026-05-05T04:53:37.86437384+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T04:53:37.864423981+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T04:53:37.864417391+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T04:53:37.864485864+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T04:53:37.864486636+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3344108-3356955-2788809641/socket","Net":"unix"}} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..438b720809e51597e5f07935dfb7754f20dbea57 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2026-05-05T04:49:12.836704419+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T04:49:13.227309076+09:00","level":"INFO","msg":"stream: created new stream","id":"lvwg9pfc"} +{"time":"2026-05-05T04:49:13.227359755+09:00","level":"INFO","msg":"handler: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:49:13.227410362+09:00","level":"INFO","msg":"stream: started","id":"lvwg9pfc"} +{"time":"2026-05-05T04:49:13.227431595+09:00","level":"INFO","msg":"writer: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:49:13.227433254+09:00","level":"INFO","msg":"sender: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:51:22.20169902+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":602} +{"time":"2026-05-05T04:51:22.202152039+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":602} +{"time":"2026-05-05T04:51:22.202456289+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":602} +{"time":"2026-05-05T04:51:23.947188792+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":602} +{"time":"2026-05-05T04:51:23.951191718+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":602} +{"time":"2026-05-05T04:51:23.951505691+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":602} +{"time":"2026-05-05T04:53:37.864421411+09:00","level":"INFO","msg":"stream: closing","id":"lvwg9pfc"} +{"time":"2026-05-05T04:53:38.535123127+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T04:53:38.873968937+09:00","level":"INFO","msg":"handler: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:53:38.874053809+09:00","level":"INFO","msg":"sender: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T04:53:38.874063427+09:00","level":"INFO","msg":"stream: closed","id":"lvwg9pfc"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b5adcb259c635dfda5f85c120fcf7aeb58e1cf86 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug.log @@ -0,0 +1,22 @@ +2026-05-05 04:49:12,586 INFO MainThread:3344108 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 04:49:12,586 INFO MainThread:3344108 [wandb_setup.py:_flush():81] Configure stats pid to 3344108 +2026-05-05 04:49:12,586 INFO MainThread:3344108 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 04:49:12,586 INFO MainThread:3344108 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug.log +2026-05-05 04:49:12,586 INFO MainThread:3344108 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/logs/debug-internal.log +2026-05-05 04:49:12,586 INFO MainThread:3344108 [wandb_init.py:init():844] calling init triggers +2026-05-05 04:49:12,586 INFO MainThread:3344108 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '900', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 04:49:12,586 INFO MainThread:3344108 [wandb_init.py:init():892] starting backend +2026-05-05 04:49:12,822 INFO MainThread:3344108 [wandb_init.py:init():895] sending inform_init request +2026-05-05 04:49:12,835 INFO MainThread:3344108 [wandb_init.py:init():903] backend started and connected +2026-05-05 04:49:12,839 INFO MainThread:3344108 [wandb_init.py:init():973] updated telemetry +2026-05-05 04:49:12,859 INFO MainThread:3344108 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 04:49:13,858 INFO MainThread:3344108 [wandb_init.py:init():1037] run resumed +2026-05-05 04:49:13,861 INFO MainThread:3344108 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 04:49:14,123 INFO MainThread:3344108 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 04:49:14,123 INFO MainThread:3344108 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 04:49:14,123 INFO MainThread:3344108 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 04:49:14,123 INFO MainThread:3344108 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 04:49:14,128 INFO MainThread:3344108 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 04:53:37,864 INFO wandb-AsyncioManager-main:3344108 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 04:53:37,864 INFO wandb-AsyncioManager-main:3344108 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/run-lvwg9pfc.wandb b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/run-lvwg9pfc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..dcbf3847479b821437a36b9d3223420bb34a56c5 Binary files /dev/null and b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_044912-lvwg9pfc/run-lvwg9pfc.wandb differ diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..911cb35dfaffdd7de17e0f25c03b086e6e00f07e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/config.yaml @@ -0,0 +1,391 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + 6zh9kmyzdkh7j025wtgawcwc8dwjekhq: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py + - '-' + - trainer.resume=True + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_t769 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=900 + - trainer.save_ckpt_iter=100 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1176008974336" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769 + startedAt: "2026-05-04T20:03:47.244974Z" + writerId: 6zh9kmyzdkh7j025wtgawcwc8dwjekhq + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "900" + offload_module_in_decoding: "False" + resume: "True" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_604_47403c800591fce4a99b.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_604_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_604_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_700_4e05a85e239f00a00e38.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_700_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_700_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_800_97779d11e00d4fea84a0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_800_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/data/real_800_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_604_b620abcbaca8ba7db5ae.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_604_b620abcbaca8ba7db5ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ece8e8ebc72aa4bf5bcd69e3a5f2e83df9e5b54 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_604_b620abcbaca8ba7db5ae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b620abcbaca8ba7db5ae8e32ae9194e2576950376926e9e6f90c5c348a8cfc7f +size 220356 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_700_7f42501ced78a5981448.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_700_7f42501ced78a5981448.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45b30c070efffb4c77c5ab8d9880c8186aa42645 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_700_7f42501ced78a5981448.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f42501ced78a5981448b14cb62b7d840086a28497d24dbf37bf93a2a03cccf5 +size 218925 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_800_6a56be7cdd484957da32.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_800_6a56be7cdd484957da32.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9d89b1f04368e90b3180c6c783c0d3932eb98fef --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/train_media/student/generation_800_6a56be7cdd484957da32.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a56be7cdd484957da3294100dc1510e28d6646918317e2af5b10224a8e8cae7 +size 242492 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_499e72e8cf7a95b7ce2a.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_499e72e8cf7a95b7ce2a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0818467bfc6790b3c9b55d8f69d6042b010baee4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_499e72e8cf7a95b7ce2a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:499e72e8cf7a95b7ce2a5287182942310160b38d174632f413e773e801fafe69 +size 261745 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_6cfbfdbe06a9c9de0d70.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_6cfbfdbe06a9c9de0d70.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9adae8f28d55a6103a64ca7a02e210fcfd1e1fd4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_6cfbfdbe06a9c9de0d70.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cfbfdbe06a9c9de0d700cd65e0278fae7a54846367511c65484d7317c4c4aa1 +size 248764 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_7153810c5617a19a2452.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_7153810c5617a19a2452.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cb2d60e20c01b972fc2493d7f752948cb28d456d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_7153810c5617a19a2452.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7153810c5617a19a2452ddd64fb09a8926cdd86913af5dc1ea19d931ab1a6100 +size 200956 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_7bfbf18b4144b09e0ca1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_7bfbf18b4144b09e0ca1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e77ce9e0f213a9a97c95a3663e268848c8b1c7e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_7bfbf18b4144b09e0ca1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bfbf18b4144b09e0ca10007499166dd33c17b07877d571c3299cfa9c368d6ae +size 206231 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_c684aef24e40913d3d79.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_c684aef24e40913d3d79.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34a0629c046d6b9239411c69136068addaba02a4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_c684aef24e40913d3d79.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c684aef24e40913d3d79fbc7d212895b2a13ac59d3fef86d0bd67a17b2034b5e +size 217712 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_d7768a50b6800410273d.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_d7768a50b6800410273d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33af3e4fbc1a0292db81a4b2634919f0c415ae0f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_d7768a50b6800410273d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7768a50b6800410273dd1a9c59772db0b9f191dc25cdadcd601f2b09d0ca3a6 +size 307483 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_e1326f3bb435a7877876.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_e1326f3bb435a7877876.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5803300301874a1b7769dc85b426e92572ad52d9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_e1326f3bb435a7877876.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1326f3bb435a7877876faf68974a4584ec74fc298cfa4f5942c44282819664d +size 188037 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_e3e5e12e7f91d944d305.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_e3e5e12e7f91d944d305.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0be6774924428b34c30a999490bbecd1b737b742 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_e3e5e12e7f91d944d305.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3e5e12e7f91d944d305896a156676f1aa3074ebbbffdd033f9bb128ea0cb3e5 +size 221972 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_ec97738b316e0805b191.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_ec97738b316e0805b191.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4bcb8114aef00ebc280d995c64e5bc76c19e39a0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_ec97738b316e0805b191.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec97738b316e0805b191569b112d011efecbd03d38432fca803e5f026fcd5c54 +size 306872 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_ff4e60c3023112769d09.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_ff4e60c3023112769d09.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3b3c18846ef4eb1fc229f73755b40b5a5e2721d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_604_ff4e60c3023112769d09.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff4e60c3023112769d09ce44f9a1b41c3105ecb974a97e5330c60556667cf235 +size 243270 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_0f9daabfb34bdab144f8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_0f9daabfb34bdab144f8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5195bac80a87ee28d65ace210da2b8e9b5586ccd --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_0f9daabfb34bdab144f8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9daabfb34bdab144f8c3818ef0e07145a8db54ae66f60b43361f6fb5ce2fbe +size 295666 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_167b48c370d4888a11a8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_167b48c370d4888a11a8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7846a6adaeda7a555ace2e138d6256fb7a9043fc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_167b48c370d4888a11a8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:167b48c370d4888a11a8b5ff5af0b0000b10c457c4a30bd198a0b67f789cd395 +size 248839 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_1c516bb19a99114dc64d.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_1c516bb19a99114dc64d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd52e2c5f13625703ab0f5b2327cdd5d31f1636a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_1c516bb19a99114dc64d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c516bb19a99114dc64d0f4fd7f08905ad23c84e019d09a5d43864744a63f2aa +size 203416 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_2c3f00aec8ff488467cd.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_2c3f00aec8ff488467cd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3fb0157f0585f377ccc2ab013cb6c46971285128 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_2c3f00aec8ff488467cd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c3f00aec8ff488467cdd40aba14a0be11b76ba544e47d539ac2f8f35b3f85b1 +size 206604 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_9786c381ce111ca85f04.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_9786c381ce111ca85f04.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07777f5b215062bfec3ad88714f8aa68e9079b5a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_9786c381ce111ca85f04.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9786c381ce111ca85f0423a805bd2d03193fe9cc902b018a2b0da07d3272f5de +size 306713 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_9a8077b0f50d01138495.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_9a8077b0f50d01138495.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7fa9b60eab9fbef719203c9b5e87f71966f19be2 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_9a8077b0f50d01138495.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a8077b0f50d011384958625ec7de4d916030e4ffc3f7fd87de52c21ab8e7c33 +size 219615 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_a6c4aa554d4fa1df2714.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_a6c4aa554d4fa1df2714.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..55c4ce20c072ea3891a5d4e21773f286d2514fcb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_a6c4aa554d4fa1df2714.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6c4aa554d4fa1df2714d7c94ad2ef3c887c114a7bb53ff183152c9f0b75715e +size 264690 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_d22b6a96303085aabe4a.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_d22b6a96303085aabe4a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c13a596baa603879b3eb5aec098bb23ccd292c53 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_d22b6a96303085aabe4a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d22b6a96303085aabe4a14b14d7682c69cfa5444a07339b481ab8cc11cd65e0d +size 245106 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_de624079a2f729993c17.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_de624079a2f729993c17.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..91201a4d8ba0025abc8ae09393e0683564c77544 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_de624079a2f729993c17.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de624079a2f729993c179394f48323aa7a527e05c0be859a72730e16f2799cbb +size 223874 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_f8d024dde8f9dfcbc78e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_f8d024dde8f9dfcbc78e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bcfb2201832c535d65e1112f6d51c3fe4ec2989c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_700_f8d024dde8f9dfcbc78e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d024dde8f9dfcbc78e55c5c37b1b5d9d92935dc561707820dfb3397adceac1 +size 186541 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_1df1b740de2839d7049d.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_1df1b740de2839d7049d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d08670e920a75cda6184f32471e4b549923c5c36 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_1df1b740de2839d7049d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1df1b740de2839d7049d46de020531f379c6824650a362059280bc4fec95803e +size 274389 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_3cb47282a0c6e17cd468.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_3cb47282a0c6e17cd468.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5b88d0a3615a3015904fa909f4b230c061647549 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_3cb47282a0c6e17cd468.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb47282a0c6e17cd468578b819b58e3b4d52e1ee5585cad49c439ebcaad5e52 +size 317133 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_662b4e119e06fcfed243.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_662b4e119e06fcfed243.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b40bce843768e3138a490baf6086450f356f3e3a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_662b4e119e06fcfed243.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:662b4e119e06fcfed243216b5d6135a9dabcb0830446fa239c8ee68abbb50604 +size 318453 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_9b8ee95c989012245e95.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_9b8ee95c989012245e95.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b4356307f90aabd414201d9c97326a03d77a9916 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_9b8ee95c989012245e95.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b8ee95c989012245e95d6cd17425c515914d93ed7696f5da19babcefee36d48 +size 194457 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_a5a88c03e4f38bb03812.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_a5a88c03e4f38bb03812.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dec89fa5b48ebc00ff478461424e2f016237ef47 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_a5a88c03e4f38bb03812.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a88c03e4f38bb03812b761c5d1a48eeb1e2c531a54571bc17d32a438df7a48 +size 217720 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_adbe9f7f90fb3f2f5d21.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_adbe9f7f90fb3f2f5d21.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3cb8a1b809a62ca71573faf00269584f1b25d315 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_adbe9f7f90fb3f2f5d21.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adbe9f7f90fb3f2f5d21fce8b14791d61ec83d5a06ce49e8db5bd81a154354e9 +size 261376 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_c60f017da4163f5d875e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_c60f017da4163f5d875e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9e3b699c186dc43f41568783b2f3c299a230968d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_c60f017da4163f5d875e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60f017da4163f5d875e460c81f50a0bc6a999d236295abcb3aa28c6c7e39c96 +size 229208 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_c721fe88bdc87fbf3873.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_c721fe88bdc87fbf3873.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11d2826875afb4ccdb8d788bdd0265c48a2533ec --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_c721fe88bdc87fbf3873.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c721fe88bdc87fbf3873ac7a32a2c46b5577491b51849af74a50462d6d27edb6 +size 253522 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_e186c4b95df8e54e6a23.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_e186c4b95df8e54e6a23.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab3528d6918028362a3930af6c91df004cb5b578 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_e186c4b95df8e54e6a23.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e186c4b95df8e54e6a23a563f2e79b3c9750892573f0055b2344da241a8bd32b +size 236575 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_f4aa7cb18093a4c80225.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_f4aa7cb18093a4c80225.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4e785b82d1d230d5f0c191227a32b38879ebbb88 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_800_f4aa7cb18093a4c80225.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4aa7cb18093a4c8022513988cc6320404937714feba49eedb76ca64b0395e15 +size 212507 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_0899e8e33a3daa243303.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_0899e8e33a3daa243303.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15fe353796a46553808484aa7755203bb31130e0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_0899e8e33a3daa243303.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0899e8e33a3daa243303a494f14a0a91825f5b23602b2df0da23105351e60df3 +size 265234 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_44d664992b102e09aee6.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_44d664992b102e09aee6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d2f001692a073e3b50e3d06440bc1b6b466edd6d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_44d664992b102e09aee6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44d664992b102e09aee6485f2a7003031afc947fea0eaac81ca7902c93da4214 +size 222605 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_564155b491ace85b197b.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_564155b491ace85b197b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8079e2c91133489050d63f3db1dd18146edcb3df --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_564155b491ace85b197b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:564155b491ace85b197b991110d854637a4a392775a5d7e296926599f5a23aea +size 234191 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_5ff7b05030a5d107b25c.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_5ff7b05030a5d107b25c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca0fcf9e4dc53066b737aeff1638f7ed1b0340c9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_5ff7b05030a5d107b25c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ff7b05030a5d107b25c647d0cc8264f270b23da9b0a2b4ea43269182fd0d1b7 +size 282342 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_6050d77106d97831a984.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_6050d77106d97831a984.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77664d591c995c5afe28c1a2a5d3eb173637d0eb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_6050d77106d97831a984.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6050d77106d97831a9847c55efb8220f211302efbcc21bee2130906bbb137ef6 +size 220475 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_7c40be6e5027d8c71d06.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_7c40be6e5027d8c71d06.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e84b2ae57f50ab665deef07530b6e39d40cee73d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_7c40be6e5027d8c71d06.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c40be6e5027d8c71d060991c9fe8055ec4e4d75b2f03e1403f92f605fbec00e +size 200029 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ac8f091c51ecec2b900d.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ac8f091c51ecec2b900d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..251c883b0c0bb3502759e3127b31d3096ca21640 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ac8f091c51ecec2b900d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac8f091c51ecec2b900d57ed9390f7bd9055c0d6a2064882ba71829f2019f092 +size 332355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_b6a1e5b3f1258ffa6f86.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_b6a1e5b3f1258ffa6f86.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e8b5a000bb026117f6cc8a4bf34b68fb53377048 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_b6a1e5b3f1258ffa6f86.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6a1e5b3f1258ffa6f8604e70a7e118b0d4aef6835a39d263b414fde7fd3c6d7 +size 256911 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ba7693264ef940cbfab8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ba7693264ef940cbfab8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51a9c61a6202de76665870e048e1154e71a1a83f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ba7693264ef940cbfab8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba7693264ef940cbfab878518913a0813486cf7c93a8c73342a8f85566e9d3af +size 242361 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ec6cb67cea81ecc6beb8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ec6cb67cea81ecc6beb8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e2ec6b6830293d82c7d1848fffaf41c72cd5cf5c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/generated_900_ec6cb67cea81ecc6beb8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec6cb67cea81ecc6beb851ecfa4e610ffa389c4529db844000d306b923edd907 +size 322596 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_604_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_700_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/media/videos/val0/reconstructed_900_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/output.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..65dc4e7269b8bf0c1f51a25ed1060342f7bf4aa0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/output.log @@ -0,0 +1,8389 @@ +[May 5, 2026 - 05:03:51 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 5, 2026 - 05:03:51 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 5, 2026 - 05:03:51 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 5, 2026 - 05:03:51 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 5, 2026 - 05:03:51 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 5, 2026 - 05:03:51 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 5, 2026 - 05:03:51 | INFO | fastgen.trainer:run:77 ] Starting training +[May 5, 2026 - 05:03:51 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 5, 2026 - 05:03:51 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 5, 2026 - 05:03:55 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 5, 2026 - 05:04:07 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 5, 2026 - 05:04:07 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 5, 2026 - 05:04:07 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 5, 2026 - 05:04:07 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 5, 2026 - 05:04:51 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 5, 2026 - 05:04:51 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 5, 2026 - 05:04:51 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 5, 2026 - 05:04:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 5, 2026 - 05:04:51 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 5, 2026 - 05:04:51 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 5, 2026 - 05:04:53 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 05:04:53 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.1s +[May 5, 2026 - 05:04:53 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 5, 2026 - 05:04:53 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 05:04:54 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 05:04:55 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.2s +[May 5, 2026 - 05:04:55 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 5, 2026 - 05:04:55 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 05:05:07 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 05:05:07 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.5s +[May 5, 2026 - 05:05:07 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 5, 2026 - 05:05:07 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 5, 2026 - 05:05:07 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 5, 2026 - 05:05:07 | INFO | fastgen.trainer:run:137 ] Loading checkpoints for resuming .. +[May 5, 2026 - 05:05:07 | INFO | fastgen.utils.checkpointer:load:436 ] Loading FSDP model from prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500 +[May 5, 2026 - 05:05:07 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key net... +[May 5, 2026 - 05:05:10 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key fake_score... +[May 5, 2026 - 05:05:12 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key net... +[May 5, 2026 - 05:05:12 | SUCCESS | fastgen.utils.checkpointer:load:544 ] Successfully loaded optimizer state for net +[May 5, 2026 - 05:05:12 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key fake_score... +[May 5, 2026 - 05:05:13 | SUCCESS | fastgen.utils.checkpointer:load:544 ] Successfully loaded optimizer state for fake_score +[May 5, 2026 - 05:05:13 | INFO | fastgen.utils.checkpointer:load:565 ] Loading the scheduler_dict... +[May 5, 2026 - 05:05:13 | INFO | fastgen.utils.checkpointer:load:573 ] Loading the gradient scaler... +[May 5, 2026 - 05:05:13 | WARNING | fastgen.utils.checkpointer:load:578 ] Gradient scaler state is empty (likely saved from disabled scaler), skipping load. +[May 5, 2026 - 05:05:13 | INFO | fastgen.utils.checkpointer:load:581 ] Loading the callbacks... +[May 5, 2026 - 05:05:13 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 500. +[May 5, 2026 - 05:05:13 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17150.48 M total params (logical). +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4299.19 M total params LOCAL on rank 0. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 1. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 2. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 3. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 05:05:18 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 05:05:18 | INFO | fastgen.trainer:run:174 ] iter_start: 500 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.426214 37.542885 37.371113 +peak_gpu_mem_gb 58.406074 58.407636 58.405553 +peak_gpu_mem_reserved_gb 60.617676 60.619141 60.617188 +util 79.500000 84.000000 76.000000 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 1.08 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 12.81 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 10.12 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 1.74 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 501-------------------- +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:06:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 501 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.600973 37.717831 37.545773 +peak_gpu_mem_gb 61.054405 61.056748 61.053624 +peak_gpu_mem_reserved_gb 62.397461 62.400391 62.396484 +util 96.500000 99.000000 90.000000 +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 48.65 seconds +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 13.36 +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 10.18 +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 1.58 +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 502-------------------- +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:06:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 502 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.601270 37.717964 37.545998 +peak_gpu_mem_gb 61.054405 61.056748 61.053624 +peak_gpu_mem_reserved_gb 62.348633 62.380859 62.337891 +util 96.750000 99.000000 90.000000 +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 46.02 seconds +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 12.05 +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 10.18 +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 1.57 +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 503-------------------- +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:07:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 503 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.601826 37.718685 37.546150 +peak_gpu_mem_gb 61.054405 61.056748 61.053624 +peak_gpu_mem_reserved_gb 62.377930 62.380859 62.376953 +util 97.250000 100.000000 95.000000 +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 45.95 seconds +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 12.03 +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 10.15 +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 1.58 +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 504-------------------- +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:08:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 504 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662662 37.778164 37.606369 +peak_gpu_mem_gb 127.092922 127.095265 127.092141 +peak_gpu_mem_reserved_gb 131.066895 131.068359 131.066406 +util 64.500000 66.000000 62.000000 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 174.43 seconds +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 78.54 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 8.45 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.44 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 505-------------------- +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 56891.5576 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6108 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 56891.5576iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1681iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1193 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0807 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 100030.6719iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.3795 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1413250.2500iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6108iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2473 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 36117.5977iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4694 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 7622.5166iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.0807 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1413250.2500iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.0030 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 54.9290 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:11:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 505 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 506 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662914 37.778725 37.606594 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.029297 65.105469 65.003906 +util 93.750000 100.000000 86.000000 +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 47.43 seconds +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 12.79 +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 10.13 +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 1.57 +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 506-------------------- +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:12:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663239 37.778728 37.607620 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.956055 64.964844 64.929688 +util 96.750000 100.000000 89.000000 +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 45.99 seconds +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 12.06 +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 10.14 +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 1.58 +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 507-------------------- +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:12:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 507 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663388 37.778736 37.607784 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.990234 65.066406 64.964844 +util 96.250000 100.000000 93.000000 +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 45.93 seconds +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 12.03 +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 10.14 +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 1.57 +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 508-------------------- +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:13:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 508 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663568 37.778736 37.608097 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.990234 65.066406 64.964844 +util 93.250000 97.000000 90.000000 +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 43.37 seconds +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 10.75 +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 10.15 +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 1.56 +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 509-------------------- +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:14:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 509 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664939 37.778797 37.610519 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.598633 133.630859 133.587891 +util 69.250000 73.000000 66.000000 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 162.89 seconds +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 72.82 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 8.41 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.40 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 510-------------------- +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 67561.4585 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6000 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 67561.4585iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7261iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2357 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9331 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 102117.6250iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 11.8380iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1051991.6250iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6000iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6285 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 77419.2266iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.5162 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1132.7136iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.7741 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 14019.8467iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.5589 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 67362.8750iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:17:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 510 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.665421 37.779419 37.610710 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.858398 64.890625 64.847656 +util 92.000000 96.000000 89.000000 +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 50.09 seconds +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 14.13 +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 10.13 +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 1.57 +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 511-------------------- +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:18:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 511 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.665997 37.780659 37.610882 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.902344 64.906250 64.890625 +util 96.750000 100.000000 89.000000 +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 45.91 seconds +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 12.03 +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 10.13 +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 1.57 +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 512-------------------- +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:18:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 512 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 513 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.666156 37.780899 37.611050 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.877930 64.910156 64.867188 +util 90.000000 94.000000 87.000000 +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 45.86 seconds +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 12.01 +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 10.13 +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 1.58 +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 513-------------------- +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:19:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.666177 37.780750 37.611084 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.892578 64.910156 64.886719 +util 97.000000 100.000000 89.000000 +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 48.54 seconds +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 13.34 +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 10.15 +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 1.57 +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 514-------------------- +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:20:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 514 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.666575 37.780647 37.611652 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.623535 133.671875 133.607422 +util 64.750000 69.000000 61.000000 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 165.33 seconds +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 74.08 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 8.40 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.40 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 515-------------------- +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 293507.3254 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5303 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 293507.3254iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3965iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7152 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8744 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 546683.8750iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.1803 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6912693.5000iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5303iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6752 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 84994.8203iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9488 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 19881.4219iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.6798 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1571.1752iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.5811 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1289.6185iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:23:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 515 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 516 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.666753 37.780891 37.611984 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.853516 64.906250 64.695312 +util 97.000000 100.000000 89.000000 +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 47.47 seconds +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 12.82 +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 10.13 +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 1.57 +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 516-------------------- +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:23:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.666821 37.780914 37.612175 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.765625 64.812500 64.750000 +util 97.000000 99.000000 95.000000 +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 45.95 seconds +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 12.06 +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 10.13 +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 1.57 +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 517-------------------- +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:24:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 517 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667164 37.780998 37.612198 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.746094 64.792969 64.730469 +util 92.250000 96.000000 89.000000 +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 45.94 seconds +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 12.04 +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 10.14 +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 1.57 +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 518-------------------- +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:25:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 518 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667282 37.781025 37.612480 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.799805 64.832031 64.789062 +util 97.000000 100.000000 89.000000 +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 45.92 seconds +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 12.03 +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 10.15 +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 1.57 +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 519-------------------- +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:26:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 519 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667187 37.779522 37.612904 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.608398 133.669922 133.587891 +util 62.000000 74.000000 54.000000 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 165.65 seconds +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 74.22 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 8.40 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.40 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 520-------------------- +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 292237.6433 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5508 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 292237.6433iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5943iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2245 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1818 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 618449.4375iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 85.5490iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 12782241.0000iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5508iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7110 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 91307.6094iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.3059 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5496.4956iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8421 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 118684.6328iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 8.1818 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 12782241.0000iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:28:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 520 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667306 37.779629 37.612961 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.790039 64.851562 64.769531 +util 94.000000 100.000000 87.000000 +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 47.56 seconds +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 12.85 +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 10.14 +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 1.57 +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 521-------------------- +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:29:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 521 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667716 37.781010 37.613068 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.824219 64.988281 64.769531 +util 98.000000 99.000000 96.000000 +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 45.95 seconds +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.00 +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 12.07 +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 10.13 +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 1.56 +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 522-------------------- +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:30:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 522 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667796 37.781075 37.613113 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.848633 65.085938 64.769531 +util 95.000000 99.000000 92.000000 +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 45.95 seconds +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 12.07 +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 10.13 +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 1.57 +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 523-------------------- +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:31:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 523 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667773 37.780876 37.613190 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.804688 64.910156 64.769531 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 524 : avg iteration time 43.25 seconds +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 524 : data loading time 0.00 +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 524 : avg forward pass time 10.70 +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 524 : backward pass time 10.15 +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 524 : optimizer step time 1.58 +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 524-------------------- +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:31:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 524 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667686 37.780022 37.613384 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.649414 133.652344 133.648438 +util 58.000000 61.000000 55.000000 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 525 : avg iteration time 162.90 seconds +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 525 : data loading time 0.00 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 525 : avg forward pass time 72.85 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 525 : backward pass time 8.39 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 525 : optimizer step time 0.40 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 525-------------------- +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 111669.5964 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4936 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 111669.5964iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.0420iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5403 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7093 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 267822.4688iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.9462 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4968591.0000iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4936iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.4672 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 138.9858iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 0.6563 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3.7157 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8525 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 121164.9844iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.4510 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 401123.7188iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:34:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 525 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667795 37.780140 37.613491 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.746094 64.789062 64.617188 +util 89.250000 93.000000 86.000000 +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 526 : avg iteration time 47.60 seconds +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 526 : data loading time 0.00 +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 526 : avg forward pass time 12.87 +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 526 : backward pass time 10.14 +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 526 : optimizer step time 1.57 +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 526-------------------- +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:35:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 526 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668193 37.781429 37.613644 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.594727 64.597656 64.593750 +util 91.250000 95.000000 88.000000 +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 527 : avg iteration time 46.02 seconds +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 527 : data loading time 0.00 +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 527 : avg forward pass time 12.07 +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 527 : backward pass time 10.15 +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 527 : optimizer step time 1.58 +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 527-------------------- +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:36:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 527 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668242 37.781467 37.613701 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.565430 64.597656 64.554688 +util 90.750000 94.000000 88.000000 +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 528 : avg iteration time 48.59 seconds +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 528 : data loading time 0.00 +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 528 : avg forward pass time 13.35 +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 528 : backward pass time 10.15 +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 528 : optimizer step time 1.58 +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 528-------------------- +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:37:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 528 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668262 37.781467 37.613804 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.575195 64.636719 64.554688 +util 89.750000 93.000000 87.000000 +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 529 : avg iteration time 45.99 seconds +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 529 : data loading time 0.00 +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 529 : avg forward pass time 12.06 +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 529 : backward pass time 10.15 +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 529 : optimizer step time 1.57 +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 529-------------------- +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:37:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 529 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.667956 37.779827 37.613964 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.589355 133.652344 133.568359 +util 68.000000 71.000000 65.000000 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 530 : avg iteration time 165.71 seconds +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 530 : data loading time 0.00 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 530 : avg forward pass time 74.26 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 530 : backward pass time 8.43 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 530 : optimizer step time 0.40 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 530-------------------- +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 705799.3496 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5179 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 705799.3496iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8670iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.8153 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.7392 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1512171.2500iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 37.7370iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 38973692.0000iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5179iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.9775 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1149694.2500iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.8968 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 132405.5625iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7449 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1789.6182iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2339 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 259831.0156iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:40:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 530 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668151 37.780136 37.614319 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.887695 64.949219 64.867188 +util 96.750000 99.000000 90.000000 +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 531 : avg iteration time 50.24 seconds +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 531 : data loading time 0.00 +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 531 : avg forward pass time 14.20 +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 531 : backward pass time 10.13 +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 531 : optimizer step time 1.58 +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 531-------------------- +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:41:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 531 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668530 37.781624 37.614361 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.848633 65.085938 64.769531 +util 93.250000 97.000000 90.000000 +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 532 : avg iteration time 43.28 seconds +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 532 : data loading time 0.00 +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 532 : avg forward pass time 10.72 +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 532 : backward pass time 10.13 +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 532 : optimizer step time 1.57 +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 532-------------------- +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:42:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 532 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 533 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668638 37.781864 37.614407 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.716797 64.750000 64.617188 +util 95.250000 100.000000 89.000000 +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 533 : avg iteration time 43.34 seconds +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 533 : data loading time 0.00 +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 533 : avg forward pass time 10.75 +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 533 : backward pass time 10.14 +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 533 : optimizer step time 1.58 +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 533-------------------- +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668633 37.781864 37.614414 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 64.828125 64.617188 +util 90.750000 94.000000 88.000000 +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 534 : avg iteration time 46.03 seconds +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 534 : data loading time 0.00 +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 534 : avg forward pass time 12.09 +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 534 : backward pass time 10.14 +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 534 : optimizer step time 1.56 +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 534-------------------- +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:43:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 534 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 535 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668137 37.779739 37.614418 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.638184 133.646484 133.613281 +util 61.000000 73.000000 52.000000 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 535 : avg iteration time 162.98 seconds +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 535 : data loading time 0.00 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 535 : avg forward pass time 72.91 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 535 : backward pass time 8.39 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 535 : optimizer step time 0.42 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 535-------------------- +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 194124.9492 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5290 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 194124.9492iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7901iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3132 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4263 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 345449.9688iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 13.8249iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2821099.7500iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5290iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.1409 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1594014.1250iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.3132 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 13.8249 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.6797 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 85775.3203iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.0705 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3432.3447iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:46:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668258 37.780231 37.614407 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.667969 65.007812 64.554688 +util 95.000000 100.000000 89.000000 +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 536 : avg iteration time 44.99 seconds +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 536 : data loading time 0.00 +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 536 : avg forward pass time 11.55 +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 536 : backward pass time 10.14 +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 536 : optimizer step time 1.58 +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 536-------------------- +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:47:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 536 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 537 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668635 37.781704 37.614407 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.692383 65.105469 64.554688 +util 97.500000 100.000000 95.000000 +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 537 : avg iteration time 45.90 seconds +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 537 : data loading time 0.00 +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 537 : avg forward pass time 12.02 +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 537 : backward pass time 10.15 +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 537 : optimizer step time 1.57 +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 537-------------------- +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:47:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668682 37.781750 37.614422 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.643555 64.910156 64.554688 +util 94.500000 100.000000 88.000000 +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 538 : avg iteration time 43.48 seconds +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 538 : data loading time 0.00 +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 538 : avg forward pass time 10.81 +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 538 : backward pass time 10.16 +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 538 : optimizer step time 1.56 +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 538-------------------- +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:48:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 538 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668937 37.782635 37.614536 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.653320 64.949219 64.554688 +util 92.250000 96.000000 89.000000 +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 539 : avg iteration time 48.65 seconds +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 539 : data loading time 0.00 +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 539 : avg forward pass time 13.38 +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 539 : backward pass time 10.16 +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 539 : optimizer step time 1.57 +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 539-------------------- +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:49:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 539 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668442 37.780575 37.614548 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.647949 133.666016 133.593750 +util 65.000000 68.000000 62.000000 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 540 : avg iteration time 165.46 seconds +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 540 : data loading time 0.00 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 540 : avg forward pass time 74.10 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 540 : backward pass time 8.45 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 540 : optimizer step time 0.41 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 540-------------------- +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 338767.3057 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5427 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 338767.3057iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6586iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1983 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1905 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 592540.9375iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.9854iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 13007328.0000iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5427iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.5626 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 9183.6875iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4618 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 7506.7168iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.2140 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 618.9676iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.8521 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 16385.3594iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:52:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 540 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668485 37.780670 37.614548 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.824219 65.105469 64.730469 +util 98.250000 100.000000 96.000000 +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 541 : avg iteration time 50.22 seconds +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 541 : data loading time 0.00 +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 541 : avg forward pass time 14.17 +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 541 : backward pass time 10.15 +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 541 : optimizer step time 1.57 +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 541-------------------- +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:53:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 541 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668888 37.782154 37.614571 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.736328 64.871094 64.691406 +util 96.000000 99.000000 93.000000 +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 542 : avg iteration time 48.57 seconds +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 542 : data loading time 0.00 +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 542 : avg forward pass time 13.35 +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 542 : backward pass time 10.16 +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 542 : optimizer step time 1.57 +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 542-------------------- +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:53:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 542 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669100 37.782898 37.614651 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.898438 66.253906 64.832031 +util 96.750000 99.000000 94.000000 +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 543 : avg iteration time 48.61 seconds +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 543 : data loading time 0.00 +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 543 : avg forward pass time 13.37 +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 543 : backward pass time 10.15 +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 543 : optimizer step time 1.56 +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 543-------------------- +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:54:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 543 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669226 37.783386 37.614651 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.809570 65.105469 64.710938 +util 99.250000 100.000000 98.000000 +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 544 : avg iteration time 48.64 seconds +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 544 : data loading time 0.00 +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 544 : avg forward pass time 13.39 +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 544 : backward pass time 10.14 +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 544 : optimizer step time 1.58 +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 544-------------------- +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:55:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 544 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668586 37.780613 37.614693 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.672363 133.685547 133.632812 +util 67.000000 71.000000 64.000000 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 545 : avg iteration time 162.78 seconds +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 545 : data loading time 0.00 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 545 : avg forward pass time 72.78 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 545 : backward pass time 8.39 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 545 : optimizer step time 0.40 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 545-------------------- +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 692107.0889 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5190 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 692107.0889iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1973iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3085 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5373 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1425534.1250iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 13.6952iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 26026630.0000iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5190iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2218 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 253609.2812iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.5571 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1229.2982iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1874 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 236752.1875iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.8042 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 812971.8750iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:58:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 545 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668618 37.780651 37.614754 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.000000 65.046875 64.984375 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 546 : avg iteration time 50.26 seconds +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 546 : data loading time 0.00 +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 546 : avg forward pass time 14.20 +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 546 : backward pass time 10.13 +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 546 : optimizer step time 1.58 +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 546-------------------- +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:58:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 546 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.668932 37.781895 37.614765 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.809570 64.867188 64.636719 +util 90.500000 94.000000 87.000000 +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 547 : avg iteration time 48.65 seconds +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 547 : data loading time 0.00 +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 547 : avg forward pass time 13.39 +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 547 : backward pass time 10.14 +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 547 : optimizer step time 1.58 +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 547-------------------- +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 547 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669069 37.782440 37.614765 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.902344 65.066406 64.847656 +util 99.000000 100.000000 97.000000 +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 548 : avg iteration time 45.92 seconds +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 548 : data loading time 0.00 +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 548 : avg forward pass time 12.02 +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 548 : backward pass time 10.15 +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 548 : optimizer step time 1.57 +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 548-------------------- +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:00:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 548 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 549 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669242 37.783073 37.614845 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.004883 65.066406 64.984375 +util 92.250000 96.000000 89.000000 +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 549 : avg iteration time 48.50 seconds +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 549 : data loading time 0.00 +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 549 : avg forward pass time 13.32 +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 549 : backward pass time 10.16 +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 549 : optimizer step time 1.57 +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 549-------------------- +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669287 37.782970 37.614876 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.644531 133.648438 133.632812 +util 69.000000 72.000000 66.000000 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 550 : avg iteration time 165.72 seconds +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 550 : data loading time 0.00 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 550 : avg forward pass time 74.20 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 550 : backward pass time 8.42 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 550 : optimizer step time 0.42 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 550-------------------- +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 161985.1821 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4873 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 161985.1821iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6779iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1549 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8921 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 397283.1250iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.0733iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7160894.0000iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4873iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.2474 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1972454.3750iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.0607 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 455.4910iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.1549 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 10.0733 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.9785 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 386.4626iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:04:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 550 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669334 37.783173 37.614868 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.951172 64.968750 64.945312 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 551 : avg iteration time 47.58 seconds +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 551 : data loading time 0.00 +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 551 : avg forward pass time 12.86 +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 551 : backward pass time 10.13 +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 551 : optimizer step time 1.57 +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 551-------------------- +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:04:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 551 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669470 37.783669 37.614929 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.004883 65.125000 64.964844 +util 99.000000 100.000000 98.000000 +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 552 : avg iteration time 43.41 seconds +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 552 : data loading time 0.00 +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 552 : avg forward pass time 10.76 +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 552 : backward pass time 10.14 +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 552 : optimizer step time 1.59 +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 552-------------------- +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:05:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 552 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 553 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669554 37.783779 37.614906 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.838867 65.105469 64.750000 +util 98.250000 100.000000 96.000000 +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 553 : avg iteration time 43.39 seconds +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 553 : data loading time 0.00 +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 553 : avg forward pass time 10.76 +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 553 : backward pass time 10.14 +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 553 : optimizer step time 1.57 +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 553-------------------- +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:06:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669372 37.783005 37.614929 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.746094 64.851562 64.710938 +util 94.000000 97.000000 91.000000 +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 554 : avg iteration time 48.54 seconds +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 554 : data loading time 0.00 +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 554 : avg forward pass time 13.34 +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 554 : backward pass time 10.14 +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 554 : optimizer step time 1.57 +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 554-------------------- +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:07:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 554 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669166 37.781864 37.614929 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.644043 133.648438 133.630859 +util 67.500000 73.000000 58.000000 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 555 : avg iteration time 165.30 seconds +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 555 : data loading time 0.00 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 555 : avg forward pass time 74.03 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 555 : backward pass time 8.43 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 555 : optimizer step time 0.40 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 555-------------------- +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 54580.3373 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5491 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 54580.3373iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2645iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2988 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9341 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 95945.9219iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 13.4306iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1054116.7500iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5491iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.9341 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1054116.7500iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.7588 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 33.7059 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3586 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 6107.5410iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.8767 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2329.3035iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:09:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 555 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669221 37.782089 37.614883 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.848633 64.906250 64.675781 +util 98.000000 100.000000 96.000000 +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 556 : avg iteration time 47.55 seconds +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 556 : data loading time 0.00 +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 556 : avg forward pass time 12.85 +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 556 : backward pass time 10.14 +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 556 : optimizer step time 1.58 +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 556-------------------- +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:10:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 556 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669505 37.783203 37.614883 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.819336 65.027344 64.750000 +util 90.750000 94.000000 88.000000 +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 557 : avg iteration time 45.90 seconds +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 557 : data loading time 0.00 +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 557 : avg forward pass time 12.05 +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 557 : backward pass time 10.12 +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 557 : optimizer step time 1.57 +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 557-------------------- +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:11:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 557 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 558 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669499 37.783203 37.614883 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.716797 64.734375 64.710938 +util 97.750000 99.000000 95.000000 +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 558 : avg iteration time 43.30 seconds +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 558 : data loading time 0.00 +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 558 : avg forward pass time 10.72 +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 558 : backward pass time 10.13 +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 558 : optimizer step time 1.58 +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 558-------------------- +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:12:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669428 37.782841 37.614899 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.746094 64.750000 64.734375 +util 95.000000 98.000000 92.000000 +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 559 : avg iteration time 48.55 seconds +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 559 : data loading time 0.00 +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 559 : avg forward pass time 13.35 +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 559 : backward pass time 10.14 +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 559 : optimizer step time 1.56 +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 559-------------------- +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:13:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 559 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669266 37.781975 37.614918 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.619629 133.708984 133.589844 +util 61.000000 63.000000 58.000000 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 560 : avg iteration time 168.16 seconds +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 560 : data loading time 0.00 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 560 : avg forward pass time 75.48 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 560 : backward pass time 8.41 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 560 : optimizer step time 0.40 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 560-------------------- +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 43191.3901 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5646 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 43191.3901iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3766iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5278 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.8892 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 69720.3672iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 21.2339iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 963498.2500iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5646iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.6657 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 11286.4258iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9921 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 21681.3574iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1452 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 539.3369iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.2659 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 5073.2764iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 560 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669375 37.782452 37.614902 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.995117 65.085938 64.964844 +util 94.250000 99.000000 88.000000 +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 561 : avg iteration time 47.55 seconds +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 561 : data loading time 0.00 +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 561 : avg forward pass time 12.85 +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 561 : backward pass time 10.15 +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 561 : optimizer step time 1.57 +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 561-------------------- +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:16:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 561 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669514 37.782974 37.614902 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.902344 64.964844 64.714844 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 562 : avg iteration time 43.33 seconds +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 562 : data loading time 0.00 +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 562 : avg forward pass time 10.71 +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 562 : backward pass time 10.17 +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 562 : optimizer step time 1.58 +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 562-------------------- +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:17:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 562 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669526 37.782951 37.614918 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.599609 64.792969 64.535156 +util 96.750000 99.000000 94.000000 +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 563 : avg iteration time 48.59 seconds +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 563 : data loading time 0.00 +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 563 : avg forward pass time 13.36 +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 563 : backward pass time 10.15 +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 563 : optimizer step time 1.57 +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 563-------------------- +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:18:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 563 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 564 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669456 37.782677 37.614937 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.604492 64.812500 64.535156 +util 89.750000 93.000000 86.000000 +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 564 : avg iteration time 43.31 seconds +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 564 : data loading time 0.00 +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 564 : avg forward pass time 10.73 +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 564 : backward pass time 10.14 +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 564 : optimizer step time 1.58 +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 564-------------------- +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:18:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669257 37.781723 37.614971 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.583984 133.630859 133.568359 +util 67.500000 73.000000 58.000000 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 565 : avg iteration time 163.17 seconds +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 565 : data loading time 0.00 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 565 : avg forward pass time 72.96 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 565 : backward pass time 8.44 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 565 : optimizer step time 0.40 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 565-------------------- +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 237704.3965 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4980 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 237704.3965iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7202iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5297 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8629 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 564369.6250iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 21.3154iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6754944.5000iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4980iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.4965 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1088.9878iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9162 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 18628.8223iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1664 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4158.1943iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.0830 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3519.1799iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:21:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 565 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 566 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669256 37.781723 37.614979 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.702148 64.792969 64.671875 +util 93.250000 96.000000 90.000000 +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 566 : avg iteration time 47.48 seconds +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 566 : data loading time 0.00 +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 566 : avg forward pass time 12.82 +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 566 : backward pass time 10.15 +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 566 : optimizer step time 1.55 +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 566-------------------- +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:22:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669568 37.782974 37.614956 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 64.851562 64.750000 +util 91.750000 95.000000 89.000000 +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 567 : avg iteration time 45.98 seconds +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 567 : data loading time 0.00 +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 567 : avg forward pass time 12.06 +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 567 : backward pass time 10.14 +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 567 : optimizer step time 1.58 +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 567-------------------- +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:23:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 567 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669632 37.783226 37.614956 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.790039 64.851562 64.769531 +util 93.500000 100.000000 86.000000 +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 568 : avg iteration time 45.93 seconds +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 568 : data loading time 0.00 +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 568 : avg forward pass time 12.03 +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 568 : backward pass time 10.14 +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 568 : optimizer step time 1.59 +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 568-------------------- +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:23:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 568 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669638 37.783234 37.614960 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.741211 64.832031 64.710938 +util 91.000000 94.000000 88.000000 +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 569 : avg iteration time 43.46 seconds +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 569 : data loading time 0.00 +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 569 : avg forward pass time 10.78 +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 569 : backward pass time 10.15 +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 569 : optimizer step time 1.57 +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 569-------------------- +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:24:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 569 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669195 37.781174 37.615067 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.574219 133.591797 133.568359 +util 60.750000 64.000000 57.000000 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 570 : avg iteration time 165.63 seconds +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 570 : data loading time 0.00 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 570 : avg forward pass time 74.21 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 570 : backward pass time 8.41 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 570 : optimizer step time 0.40 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 570-------------------- +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 218458.8320 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5237 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 218458.8320iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4788iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4830 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1702 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 561618.8125iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 19.4154iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 12488428.0000iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5237iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.6397 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 196.2588iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0116 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3051.0090iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.4830 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 19.4154 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.3862 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 6453.0020iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:27:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 570 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669251 37.781414 37.615097 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.755859 64.789062 64.656250 +util 89.250000 92.000000 86.000000 +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 571 : avg iteration time 50.17 seconds +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 571 : data loading time 0.00 +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 571 : avg forward pass time 14.15 +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 571 : backward pass time 10.14 +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 571 : optimizer step time 1.57 +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 571-------------------- +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:28:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 571 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669559 37.782635 37.615074 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.824219 64.929688 64.789062 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 572 : avg iteration time 43.50 seconds +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 572 : data loading time 0.00 +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 572 : avg forward pass time 10.81 +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 572 : backward pass time 10.16 +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 572 : optimizer step time 1.57 +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 572-------------------- +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:28:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 572 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669599 37.782772 37.615097 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.809570 64.871094 64.789062 +util 94.750000 100.000000 89.000000 +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 573 : avg iteration time 45.89 seconds +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 573 : data loading time 0.00 +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 573 : avg forward pass time 12.02 +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 573 : backward pass time 10.12 +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 573 : optimizer step time 1.57 +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 573-------------------- +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:29:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 573 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669608 37.782772 37.615131 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.804688 64.851562 64.789062 +util 97.250000 99.000000 95.000000 +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 574 : avg iteration time 45.91 seconds +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 574 : data loading time 0.00 +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 574 : avg forward pass time 12.02 +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 574 : backward pass time 10.15 +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 574 : optimizer step time 1.57 +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 574-------------------- +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:30:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 574 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669456 37.781815 37.615273 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.613281 133.689453 133.587891 +util 60.750000 64.000000 57.000000 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 575 : avg iteration time 165.35 seconds +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 575 : data loading time 0.00 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 575 : avg forward pass time 74.04 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 575 : backward pass time 8.44 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 575 : optimizer step time 0.41 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 575-------------------- +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 628102.0088 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5292 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 628102.0088iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.2379iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1969 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.4291 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1227003.7500iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.9548iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 20962104.0000iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5292iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.4299 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 52042.3320iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1142 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3745.8757iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.8721 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 312.3450iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2491 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 36246.7148iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:33:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 575 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 576 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669595 37.782326 37.615318 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.814453 64.890625 64.789062 +util 98.500000 100.000000 97.000000 +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 576 : avg iteration time 47.67 seconds +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 576 : data loading time 0.00 +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 576 : avg forward pass time 12.92 +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 576 : backward pass time 10.13 +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 576 : optimizer step time 1.57 +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 576-------------------- +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:34:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.669997 37.783825 37.615429 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.750977 64.753906 64.750000 +util 96.500000 99.000000 94.000000 +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 577 : avg iteration time 45.96 seconds +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 577 : data loading time 0.00 +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 577 : avg forward pass time 12.05 +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 577 : backward pass time 10.15 +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 577 : optimizer step time 1.56 +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 577-------------------- +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:34:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 577 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670077 37.784122 37.615406 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.838867 64.988281 64.789062 +util 95.000000 98.000000 92.000000 +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 578 : avg iteration time 43.39 seconds +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 578 : data loading time 0.00 +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 578 : avg forward pass time 10.78 +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 578 : backward pass time 10.13 +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 578 : optimizer step time 1.57 +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 578-------------------- +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:35:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 578 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670120 37.784218 37.615490 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 64.968750 64.710938 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 579 : avg iteration time 45.85 seconds +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 579 : data loading time 0.00 +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 579 : avg forward pass time 12.02 +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 579 : backward pass time 10.12 +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 579 : optimizer step time 1.57 +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 579-------------------- +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:36:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 579 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670372 37.782608 37.616203 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.647461 133.650391 133.646484 +util 60.000000 63.000000 57.000000 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 580 : avg iteration time 160.10 seconds +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 580 : data loading time 0.00 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 580 : avg forward pass time 71.45 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 580 : backward pass time 8.43 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 580 : optimizer step time 0.40 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 580-------------------- +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 751625.7090 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4846 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 751625.7090iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4649iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5710 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6641 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1476867.5000iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.1329 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 33538966.0000iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4846iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.5943 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3.2826 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.8626 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 6751028.5000iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.3272 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 313109.5312iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.7024 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1643.7125iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:38:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 580 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670379 37.782619 37.616203 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.931641 65.125000 64.867188 +util 94.000000 99.000000 88.000000 +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 581 : avg iteration time 47.55 seconds +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 581 : data loading time 0.00 +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 581 : avg forward pass time 12.85 +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 581 : backward pass time 10.13 +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 581 : optimizer step time 1.57 +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 581-------------------- +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:39:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 581 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670820 37.784382 37.616203 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.829102 65.066406 64.750000 +util 97.750000 100.000000 95.000000 +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 582 : avg iteration time 43.33 seconds +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 582 : data loading time 0.00 +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 582 : avg forward pass time 10.75 +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 582 : backward pass time 10.13 +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 582 : optimizer step time 1.57 +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 582-------------------- +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:40:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 582 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671015 37.785160 37.616203 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.814453 65.066406 64.730469 +util 95.000000 98.000000 92.000000 +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 583 : avg iteration time 45.96 seconds +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 583 : data loading time 0.00 +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 583 : avg forward pass time 12.05 +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 583 : backward pass time 10.14 +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 583 : optimizer step time 1.57 +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 583-------------------- +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:41:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 583 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671027 37.785137 37.616253 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.843750 65.066406 64.769531 +util 92.250000 99.000000 88.000000 +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 584 : avg iteration time 46.13 seconds +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 584 : data loading time 0.00 +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 584 : avg forward pass time 12.12 +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 584 : backward pass time 10.16 +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 584 : optimizer step time 1.57 +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 584-------------------- +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:42:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 584 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670456 37.782795 37.616253 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.603516 133.607422 133.591797 +util 64.000000 67.000000 61.000000 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 585 : avg iteration time 168.08 seconds +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 585 : data loading time 0.00 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 585 : avg forward pass time 75.44 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 585 : backward pass time 8.44 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 585 : optimizer step time 0.40 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 585-------------------- +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 137340.4988 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5506 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 137340.4988iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5545iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4300 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7095 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 229073.7031iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 17.4632iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4970273.0000iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5506iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2305 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 258091.4219iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.6111 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 185.3352iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.7972 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 268.9025iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.0245 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 23130.9160iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:44:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 585 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670505 37.783043 37.616253 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.760742 64.910156 64.710938 +util 89.500000 92.000000 86.000000 +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 586 : avg iteration time 47.73 seconds +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 586 : data loading time 0.00 +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 586 : avg forward pass time 12.93 +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 586 : backward pass time 10.16 +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 586 : optimizer step time 1.57 +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 586-------------------- +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:45:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 586 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670941 37.784760 37.616257 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.785156 64.949219 64.730469 +util 91.000000 94.000000 88.000000 +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 587 : avg iteration time 48.62 seconds +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 587 : data loading time 0.00 +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 587 : avg forward pass time 13.38 +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 587 : backward pass time 10.14 +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 587 : optimizer step time 1.57 +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 587-------------------- +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 587 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671126 37.785515 37.616268 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.619141 64.632812 64.578125 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 588 : avg iteration time 45.88 seconds +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 588 : data loading time 0.00 +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 588 : avg forward pass time 12.02 +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 588 : backward pass time 10.14 +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 588 : optimizer step time 1.57 +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 588-------------------- +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:47:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 588 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671229 37.785870 37.616272 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.092773 65.160156 64.890625 +util 92.500000 100.000000 88.000000 +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 589 : avg iteration time 45.89 seconds +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 589 : data loading time 0.00 +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 589 : avg forward pass time 12.01 +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 589 : backward pass time 10.14 +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 589 : optimizer step time 1.58 +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 589-------------------- +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:47:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 589 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670710 37.783268 37.616585 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.574219 133.591797 133.568359 +util 65.500000 68.000000 62.000000 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 590 : avg iteration time 167.89 seconds +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 590 : data loading time 0.00 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 590 : avg forward pass time 75.34 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 590 : backward pass time 8.40 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 590 : optimizer step time 0.40 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 590-------------------- +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 291616.0427 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5099 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 291616.0427iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7524iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6408 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0303 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 543315.7500iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 26.6175iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9440795.0000iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5099iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.0714 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 465.3908iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.2257 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1888630.6250iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6259 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 10423.6943iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2373 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 261609.0625iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:50:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 590 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 591 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670710 37.783272 37.616604 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.531250 64.636719 64.496094 +util 98.500000 100.000000 97.000000 +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 591 : avg iteration time 47.63 seconds +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 591 : data loading time 0.00 +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 591 : avg forward pass time 12.88 +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 591 : backward pass time 10.17 +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 591 : optimizer step time 1.57 +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 591-------------------- +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:51:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671094 37.784775 37.616627 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.667969 65.125000 64.515625 +util 90.250000 93.000000 87.000000 +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 592 : avg iteration time 43.29 seconds +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 592 : data loading time 0.00 +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 592 : avg forward pass time 10.71 +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 592 : backward pass time 10.15 +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 592 : optimizer step time 1.57 +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 592-------------------- +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:52:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 592 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671431 37.786030 37.616608 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.853516 64.925781 64.636719 +util 96.500000 100.000000 88.000000 +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 593 : avg iteration time 43.34 seconds +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 593 : data loading time 0.00 +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 593 : avg forward pass time 10.75 +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 593 : backward pass time 10.16 +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 593 : optimizer step time 1.56 +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 593-------------------- +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:52:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 593 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671550 37.786530 37.616608 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.619141 64.988281 64.496094 +util 92.500000 95.000000 90.000000 +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 594 : avg iteration time 48.48 seconds +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 594 : data loading time 0.00 +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 594 : avg forward pass time 13.30 +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 594 : backward pass time 10.15 +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 594 : optimizer step time 1.58 +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 594-------------------- +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:53:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 594 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670874 37.783649 37.616634 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.647461 133.650391 133.646484 +util 64.000000 72.000000 57.000000 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 595 : avg iteration time 165.54 seconds +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 595 : data loading time 0.00 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 595 : avg forward pass time 74.15 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 595 : backward pass time 8.39 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 595 : optimizer step time 0.40 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 595-------------------- +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 359450.2427 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4975 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 359450.2427iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4687iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0733 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2441 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 527092.7500iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.5559 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 14478045.0000iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4975iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.5273 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 21.2115 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.6304 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1423.4183iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2624 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 5038.3008iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.0965 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 26716.1016iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:56:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +wandb: WARNING Tried to log to step 595 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 596 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.670953 37.783901 37.616623 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.965820 65.085938 64.925781 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 596 : avg iteration time 50.14 seconds +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 596 : data loading time 0.00 +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 596 : avg forward pass time 14.13 +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 596 : backward pass time 10.15 +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 596 : optimizer step time 1.58 +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 596-------------------- +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:57:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671450 37.785908 37.616627 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.755859 64.773438 64.750000 +util 98.250000 100.000000 96.000000 +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 597 : avg iteration time 46.00 seconds +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 597 : data loading time 0.00 +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 597 : avg forward pass time 12.04 +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 597 : backward pass time 10.16 +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 597 : optimizer step time 1.60 +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 597-------------------- +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:58:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 597 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 598 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671529 37.786167 37.616661 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.716797 64.734375 64.710938 +util 96.500000 99.000000 93.000000 +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 598 : avg iteration time 43.35 seconds +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 598 : data loading time 0.00 +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 598 : avg forward pass time 10.74 +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 598 : backward pass time 10.15 +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 598 : optimizer step time 1.58 +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 598-------------------- +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:58:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671606 37.786430 37.616688 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.913086 66.273438 64.832031 +util 94.000000 100.000000 88.000000 +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 599 : avg iteration time 48.68 seconds +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 599 : data loading time 0.00 +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 599 : avg forward pass time 13.40 +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 599 : backward pass time 10.14 +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 599 : optimizer step time 1.58 +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 599-------------------- +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 599 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.671093 37.784256 37.616673 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.608398 133.611328 133.607422 +util 64.750000 72.000000 58.000000 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 600 : avg iteration time 160.21 seconds +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 600 : data loading time 0.00 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 600 : avg forward pass time 71.52 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 600 : backward pass time 8.41 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 600 : optimizer step time 0.40 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 600-------------------- +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 67094.4106 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4584 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 67094.4106iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7450iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0936 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9702 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 159346.8125iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.9101 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1133012.6250iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4584iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4744 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7698.1353iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3827 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 47355.4883iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.9612 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 150600.9844iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2630 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 275381.6562iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 07:02:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +wandb: WARNING Tried to log to step 600 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 07:02:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 18.07 seconds +[May 5, 2026 - 07:02:40 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 600 +[May 5, 2026 - 07:02:40 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600 +[May 5, 2026 - 07:02:57 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 600 +[May 5, 2026 - 07:02:57 | INFO | fastgen.trainer:validate:358 ] Validation iteration 600 +[May 5, 2026 - 07:02:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:02:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 5, 2026 - 07:03:03 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 5, 2026 - 07:03:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.234 +[May 5, 2026 - 07:03:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.838 +[May 5, 2026 - 07:03:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.001 +[May 5, 2026 - 07:03:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.590 +[May 5, 2026 - 07:03:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.241 +[May 5, 2026 - 07:03:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.971 +[May 5, 2026 - 07:03:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.784 +[May 5, 2026 - 07:03:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.922 +[May 5, 2026 - 07:03:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.934 +[May 5, 2026 - 07:03:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 07:03:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 07:03:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.682 +[May 5, 2026 - 07:03:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 600-------------------- +[May 5, 2026 - 07:03:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 07:04:04 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 600 +[May 5, 2026 - 07:04:04 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.220 (n=10) +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487080 41.611027 41.429104 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.695801 68.212891 67.523438 +util 92.000000 99.000000 88.000000 +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 601 : avg iteration time 144.81 seconds +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 601 : data loading time 0.00 +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 601 : avg forward pass time 10.33 +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 601 : backward pass time 10.14 +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 601 : optimizer step time 1.58 +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 601-------------------- +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:04:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 601 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +wandb: WARNING Tried to log to step 602 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487087 41.611027 41.429127 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.725098 68.388672 67.503906 +util 96.750000 99.000000 94.000000 +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 602 : avg iteration time 46.09 seconds +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 602 : data loading time 0.00 +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 602 : avg forward pass time 12.09 +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 602 : backward pass time 10.18 +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 602 : optimizer step time 1.58 +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 602-------------------- +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:05:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487096 41.611053 41.429115 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.793457 68.544922 67.542969 +util 90.000000 93.000000 87.000000 +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 603 : avg iteration time 46.13 seconds +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 603 : data loading time 0.00 +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 603 : avg forward pass time 12.12 +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 603 : backward pass time 10.14 +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 603 : optimizer step time 1.59 +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 603-------------------- +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:06:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +wandb: WARNING Tried to log to step 603 that is less than the current step 604. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order. +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.486778 41.609783 41.429115 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.710449 68.330078 67.503906 +util 95.000000 98.000000 92.000000 +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 604 : avg iteration time 46.06 seconds +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 604 : data loading time 0.00 +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 604 : avg forward pass time 12.10 +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 604 : backward pass time 10.14 +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 604 : optimizer step time 1.58 +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 604-------------------- +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:07:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.486976 41.608036 41.429924 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.115234 136.011719 +util 64.000000 72.000000 57.000000 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 605 : avg iteration time 165.70 seconds +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 605 : data loading time 0.00 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 605 : avg forward pass time 74.20 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 605 : backward pass time 8.49 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 605 : optimizer step time 0.40 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 605-------------------- +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 90881.8544 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5284 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 90881.8544iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1365iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5720 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3343 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 193918.6250iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.1391 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2346891.0000iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5284iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.8487 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 120250.3906iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.8254 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 15534.5469iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.3343 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2346891.0000iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 0.8902 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 5.9322 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:09:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487132 41.608536 41.429943 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.354004 68.017578 67.132812 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 606 : avg iteration time 47.75 seconds +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 606 : data loading time 0.00 +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 606 : avg forward pass time 12.92 +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 606 : backward pass time 10.16 +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 606 : optimizer step time 1.56 +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 606-------------------- +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:10:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487200 41.608799 41.429951 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.549316 67.861328 67.445312 +util 94.750000 97.000000 92.000000 +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 607 : avg iteration time 43.34 seconds +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 607 : data loading time 0.00 +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 607 : avg forward pass time 10.75 +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 607 : backward pass time 10.15 +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 607 : optimizer step time 1.57 +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 607-------------------- +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:11:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487225 41.608822 41.429955 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.261230 67.880859 67.054688 +util 98.250000 100.000000 96.000000 +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 608 : avg iteration time 48.65 seconds +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 608 : data loading time 0.00 +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 608 : avg forward pass time 13.41 +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 608 : backward pass time 10.12 +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 608 : optimizer step time 1.57 +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 608-------------------- +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:12:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487222 41.608826 41.429951 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.261230 67.880859 67.054688 +util 88.750000 91.000000 87.000000 +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 609 : avg iteration time 45.89 seconds +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 609 : data loading time 0.00 +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 609 : avg forward pass time 12.02 +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 609 : backward pass time 10.13 +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 609 : optimizer step time 1.59 +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 609-------------------- +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:12:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487162 41.608444 41.429951 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.993652 136.115234 135.953125 +util 63.000000 73.000000 59.000000 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 610 : avg iteration time 159.99 seconds +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 610 : data loading time 0.00 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 610 : avg forward pass time 71.37 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 610 : backward pass time 8.44 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 610 : optimizer step time 0.40 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 610-------------------- +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 260646.7451 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4761 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 260646.7451iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.2236iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.6061 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9141 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 636286.3750iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 183.4862iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7483369.0000iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4761iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.1038 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 27106.6875iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4950 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 8022.3784iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.6828 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1580.6554iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.7563 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 738627.0625iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:15:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487267 41.608749 41.429951 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.573730 68.017578 67.425781 +util 91.500000 94.000000 88.000000 +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 611 : avg iteration time 50.23 seconds +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 611 : data loading time 0.00 +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 611 : avg forward pass time 14.19 +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 611 : backward pass time 10.14 +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 611 : optimizer step time 1.56 +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 611-------------------- +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:16:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487280 41.608772 41.429951 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500488 68.017578 67.328125 +util 94.500000 100.000000 89.000000 +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 612 : avg iteration time 45.95 seconds +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 612 : data loading time 0.00 +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 612 : avg forward pass time 12.05 +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 612 : backward pass time 10.14 +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 612 : optimizer step time 1.57 +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 612-------------------- +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:17:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487299 41.608818 41.429951 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.656738 68.056641 67.523438 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 613 : avg iteration time 43.32 seconds +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 613 : data loading time 0.00 +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 613 : avg forward pass time 10.74 +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 613 : backward pass time 10.14 +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 613 : optimizer step time 1.57 +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 613-------------------- +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:17:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487411 41.609264 41.429951 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.642090 67.998047 67.523438 +util 93.250000 96.000000 90.000000 +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 614 : avg iteration time 45.95 seconds +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 614 : data loading time 0.00 +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 614 : avg forward pass time 12.05 +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 614 : backward pass time 10.14 +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 614 : optimizer step time 1.56 +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 614-------------------- +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:18:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487404 41.609215 41.429951 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.032715 136.154297 135.992188 +util 70.250000 72.000000 67.000000 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 615 : avg iteration time 160.30 seconds +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 615 : data loading time 0.00 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 615 : avg forward pass time 71.51 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 615 : backward pass time 8.42 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 615 : optimizer step time 0.40 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 615-------------------- +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 272589.9585 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5031 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 272589.9585iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6694iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3370 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8131 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 544496.5625iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 14.4979iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6114117.5000iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5031iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.4125 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2744246.2500iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.1850 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1741217.7500iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.3008 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 99.6443 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.0002 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 54.6203 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:21:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487462 41.609459 41.429962 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.915527 67.919922 67.914062 +util 96.750000 99.000000 94.000000 +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 616 : avg iteration time 47.54 seconds +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 616 : data loading time 0.00 +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 616 : avg forward pass time 12.84 +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 616 : backward pass time 10.13 +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 616 : optimizer step time 1.58 +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 616-------------------- +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:22:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487464 41.609436 41.429962 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.358887 67.978516 67.152344 +util 89.750000 92.000000 87.000000 +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 617 : avg iteration time 45.91 seconds +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 617 : data loading time 0.00 +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 617 : avg forward pass time 12.04 +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 617 : backward pass time 10.12 +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 617 : optimizer step time 1.58 +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 617-------------------- +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:22:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487459 41.609436 41.429989 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.920410 67.933594 67.880859 +util 99.750000 100.000000 99.000000 +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 618 : avg iteration time 48.55 seconds +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 618 : data loading time 0.00 +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 618 : avg forward pass time 13.34 +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 618 : backward pass time 10.14 +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 618 : optimizer step time 1.58 +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 618-------------------- +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:23:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487481 41.609463 41.429989 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.018066 68.050781 67.919922 +util 97.750000 99.000000 95.000000 +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 619 : avg iteration time 43.36 seconds +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 619 : data loading time 0.00 +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 619 : avg forward pass time 10.76 +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 619 : backward pass time 10.13 +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 619 : optimizer step time 1.58 +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 619-------------------- +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:24:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487455 41.609322 41.429989 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.008301 136.115234 135.972656 +util 70.750000 73.000000 68.000000 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 620 : avg iteration time 168.26 seconds +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 620 : data loading time 0.00 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 620 : avg forward pass time 75.53 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 620 : backward pass time 8.41 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 620 : optimizer step time 0.41 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 620-------------------- +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 597149.9343 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5777 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 597149.9343iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7057iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2545 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.7454 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1383640.7500iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.2927iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 39463496.0000iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5777iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.2252 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4676.5928iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.3377 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 792.6401iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.9904 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 21606.0391iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9611 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 20375.7734iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:27:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487467 41.609303 41.429989 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.568848 68.056641 67.406250 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 621 : avg iteration time 47.56 seconds +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 621 : data loading time 0.00 +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 621 : avg forward pass time 12.85 +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 621 : backward pass time 10.15 +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 621 : optimizer step time 1.57 +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 621-------------------- +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:28:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487550 41.609573 41.429989 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.637207 67.978516 67.523438 +util 95.750000 98.000000 93.000000 +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 622 : avg iteration time 48.75 seconds +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 622 : data loading time 0.00 +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 622 : avg forward pass time 13.45 +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 622 : backward pass time 10.13 +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 622 : optimizer step time 1.58 +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 622-------------------- +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487556 41.609585 41.429989 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.978516 67.367188 +util 99.000000 100.000000 98.000000 +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 623 : avg iteration time 48.66 seconds +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 623 : data loading time 0.00 +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 623 : avg forward pass time 13.38 +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 623 : backward pass time 10.15 +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 623 : optimizer step time 1.57 +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 623-------------------- +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:29:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487529 41.609524 41.429989 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.978516 67.367188 +util 98.250000 99.000000 96.000000 +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 624 : avg iteration time 45.98 seconds +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 624 : data loading time 0.00 +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 624 : avg forward pass time 12.06 +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 624 : backward pass time 10.15 +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 624 : optimizer step time 1.59 +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 624-------------------- +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:30:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487362 41.608624 41.430012 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.006836 136.056641 135.990234 +util 61.750000 64.000000 58.000000 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 625 : avg iteration time 162.73 seconds +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 625 : data loading time 0.00 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 625 : avg forward pass time 72.76 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 625 : backward pass time 8.42 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 625 : optimizer step time 0.40 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 625-------------------- +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 7302733.4839 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5615 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 7302733.4839iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7143iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4488 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.9998 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 15463421.0000iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 18.1309iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 484946848.0000iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5615iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8969 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 17923.3477iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.6206 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1395.9003iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.6045 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 545222.9375iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.7893 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 106786.2734iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:33:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487375 41.608742 41.429993 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.437500 67.882812 67.289062 +util 90.000000 92.000000 87.000000 +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 626 : avg iteration time 44.96 seconds +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 626 : data loading time 0.00 +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 626 : avg forward pass time 11.55 +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 626 : backward pass time 10.14 +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 626 : optimizer step time 1.59 +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 626-------------------- +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:33:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487506 41.609226 41.429996 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.564453 67.980469 67.425781 +util 94.000000 96.000000 91.000000 +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 627 : avg iteration time 46.05 seconds +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 627 : data loading time 0.00 +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 627 : avg forward pass time 12.09 +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 627 : backward pass time 10.16 +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 627 : optimizer step time 1.57 +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 627-------------------- +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:34:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487532 41.609230 41.430019 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.579102 68.039062 67.425781 +util 95.750000 98.000000 93.000000 +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 628 : avg iteration time 46.02 seconds +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 628 : data loading time 0.00 +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 628 : avg forward pass time 12.07 +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 628 : backward pass time 10.16 +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 628 : optimizer step time 1.57 +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 628-------------------- +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:35:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487535 41.609230 41.430031 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.574219 68.019531 67.425781 +util 96.750000 99.000000 94.000000 +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 629 : avg iteration time 46.01 seconds +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 629 : data loading time 0.00 +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 629 : avg forward pass time 12.07 +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 629 : backward pass time 10.14 +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 629 : optimizer step time 1.58 +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 629-------------------- +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:36:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487459 41.608917 41.430019 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.115234 136.011719 +util 61.500000 64.000000 58.000000 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 630 : avg iteration time 168.16 seconds +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 630 : data loading time 0.00 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 630 : avg forward pass time 75.47 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 630 : backward pass time 8.43 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 630 : optimizer step time 0.40 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 630-------------------- +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1639432.0910 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4883 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1639432.0910iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3736iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5139 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.0273 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2385647.7500iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 20.6505iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 69351280.0000iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4883iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.1741 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 31201.2109iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.7373 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 96235.5469iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.9027 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 44.9391 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.5901 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 71698.5156iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:39:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487583 41.609413 41.430019 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.466309 67.998047 67.289062 +util 92.750000 95.000000 90.000000 +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 631 : avg iteration time 45.19 seconds +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 631 : data loading time 0.00 +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 631 : avg forward pass time 11.68 +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 631 : backward pass time 10.13 +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 631 : optimizer step time 1.57 +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 631-------------------- +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:39:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487853 41.610435 41.430042 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471191 67.958984 67.308594 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 632 : avg iteration time 48.68 seconds +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 632 : data loading time 0.00 +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 632 : avg forward pass time 13.42 +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 632 : backward pass time 10.14 +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 632 : optimizer step time 1.56 +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 632-------------------- +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:40:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487853 41.610435 41.430031 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.451660 67.880859 67.308594 +util 90.000000 92.000000 87.000000 +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 633 : avg iteration time 45.89 seconds +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 633 : data loading time 0.00 +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 633 : avg forward pass time 12.01 +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 633 : backward pass time 10.14 +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 633 : optimizer step time 1.58 +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 633-------------------- +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:41:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487859 41.610435 41.430054 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.466309 67.939453 67.308594 +util 96.750000 100.000000 89.000000 +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 634 : avg iteration time 43.34 seconds +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 634 : data loading time 0.00 +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 634 : avg forward pass time 10.74 +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 634 : backward pass time 10.13 +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 634 : optimizer step time 1.58 +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 634-------------------- +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:42:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487452 41.608784 41.430061 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.003418 136.037109 135.992188 +util 64.000000 72.000000 57.000000 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 635 : avg iteration time 165.63 seconds +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 635 : data loading time 0.00 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 635 : avg forward pass time 74.22 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 635 : backward pass time 8.43 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 635 : optimizer step time 0.40 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 635-------------------- +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 111573.5414 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5483 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 111573.5414iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3522iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6302 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5122 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 178125.4844iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.5271 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3349650.0000iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5483iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2169 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 33990.7461iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.3380 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 107.3358iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4293 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 7034.3096iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.7702 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 254.7992iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:44:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487514 41.609032 41.430061 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.476074 67.978516 67.308594 +util 93.500000 95.000000 91.000000 +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 636 : avg iteration time 47.47 seconds +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 636 : data loading time 0.00 +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 636 : avg forward pass time 12.80 +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 636 : backward pass time 10.14 +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 636 : optimizer step time 1.58 +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 636-------------------- +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:45:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488223 41.611855 41.430061 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.457031 67.960938 67.289062 +util 99.000000 100.000000 97.000000 +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 637 : avg iteration time 48.67 seconds +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 637 : data loading time 0.00 +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 637 : avg forward pass time 13.41 +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 637 : backward pass time 10.12 +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 637 : optimizer step time 1.57 +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 637-------------------- +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:46:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488242 41.611965 41.430038 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471680 67.960938 67.308594 +util 93.000000 95.000000 90.000000 +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 638 : avg iteration time 45.92 seconds +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 638 : data loading time 0.00 +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 638 : avg forward pass time 12.03 +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 638 : backward pass time 10.13 +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 638 : optimizer step time 1.57 +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 638-------------------- +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:47:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488252 41.611984 41.430058 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.476562 67.980469 67.308594 +util 91.000000 93.000000 88.000000 +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 639 : avg iteration time 45.98 seconds +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 639 : data loading time 0.00 +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 639 : avg forward pass time 12.04 +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 639 : backward pass time 10.14 +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 639 : optimizer step time 1.58 +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 639-------------------- +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:47:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487864 41.610451 41.430038 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.173828 135.992188 +util 60.000000 62.000000 57.000000 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 640 : avg iteration time 160.01 seconds +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 640 : data loading time 0.00 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 640 : avg forward pass time 71.39 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 640 : backward pass time 8.42 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 640 : optimizer step time 0.40 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 640-------------------- +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 62416.0166 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4756 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 62416.0166iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4928iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1515 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9627 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 138270.1875iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.0051iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1116135.2500iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4756iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2484 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 36199.1133iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6325 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 78041.8359iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.2105 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 247941.4688iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.5040 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1105.4260iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:50:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487878 41.610455 41.430061 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.886230 68.037109 67.835938 +util 94.750000 97.000000 92.000000 +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 641 : avg iteration time 50.23 seconds +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 641 : data loading time 0.00 +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 641 : avg forward pass time 14.21 +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 641 : backward pass time 10.13 +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 641 : optimizer step time 1.57 +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 641-------------------- +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:51:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488078 41.611271 41.430038 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.803711 68.000000 67.738281 +util 98.250000 100.000000 96.000000 +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 642 : avg iteration time 48.65 seconds +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 642 : data loading time 0.00 +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 642 : avg forward pass time 13.38 +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 642 : backward pass time 10.14 +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 642 : optimizer step time 1.58 +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 642-------------------- +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:52:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488121 41.611431 41.430054 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.452148 68.000000 67.269531 +util 95.000000 97.000000 92.000000 +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 643 : avg iteration time 45.96 seconds +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 643 : data loading time 0.00 +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 643 : avg forward pass time 12.04 +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 643 : backward pass time 10.14 +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 643 : optimizer step time 1.58 +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 643-------------------- +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:53:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488173 41.611645 41.430065 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.447266 67.980469 67.269531 +util 91.500000 99.000000 87.000000 +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 644 : avg iteration time 48.52 seconds +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 644 : data loading time 0.00 +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 644 : avg forward pass time 13.36 +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 644 : backward pass time 10.11 +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 644 : optimizer step time 1.56 +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 644-------------------- +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:53:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487736 41.609894 41.430065 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.080078 136.087891 136.056641 +util 61.250000 63.000000 58.000000 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 645 : avg iteration time 165.46 seconds +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 645 : data loading time 0.00 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 645 : avg forward pass time 74.12 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 645 : backward pass time 8.41 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 645 : optimizer step time 0.40 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 645-------------------- +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 25063.7771 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4949 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 25063.7771iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3747iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2250 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.4486 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 51376.5703iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 85.6267iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 399184.3750iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4949iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2097 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 33503.6211iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.2250 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 85.6267 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3957 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 6577.5786iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.4060 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 908.6311iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:56:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487793 41.610020 41.430065 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.466309 67.998047 67.289062 +util 94.000000 99.000000 89.000000 +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 646 : avg iteration time 47.53 seconds +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 646 : data loading time 0.00 +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 646 : avg forward pass time 12.84 +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 646 : backward pass time 10.14 +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 646 : optimizer step time 1.57 +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 646-------------------- +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:57:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488098 41.611107 41.430065 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500977 68.019531 67.328125 +util 99.000000 100.000000 97.000000 +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 647 : avg iteration time 48.56 seconds +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 647 : data loading time 0.00 +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 647 : avg forward pass time 13.35 +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 647 : backward pass time 10.15 +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 647 : optimizer step time 1.57 +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 647-------------------- +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:58:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488158 41.611370 41.430065 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.496094 68.000000 67.328125 +util 95.250000 97.000000 92.000000 +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 648 : avg iteration time 48.54 seconds +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 648 : data loading time 0.00 +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 648 : avg forward pass time 13.33 +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 648 : backward pass time 10.14 +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 648 : optimizer step time 1.58 +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 648-------------------- +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:59:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488217 41.611599 41.430065 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.496094 68.000000 67.328125 +util 90.250000 92.000000 87.000000 +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 649 : avg iteration time 48.74 seconds +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 649 : data loading time 0.00 +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 649 : avg forward pass time 13.44 +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 649 : backward pass time 10.15 +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 649 : optimizer step time 1.58 +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 649-------------------- +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 07:59:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487700 41.609497 41.430065 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.988770 136.095703 135.953125 +util 69.250000 71.000000 66.000000 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 650 : avg iteration time 168.14 seconds +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 650 : data loading time 0.00 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 650 : avg forward pass time 75.46 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 650 : backward pass time 8.41 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 650 : optimizer step time 0.40 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 650-------------------- +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 95404.1807 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4905 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 95404.1807iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5615iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1346 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4399 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 186909.9062iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.6722 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2898720.0000iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4905iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0540 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3321.1316iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.1639 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 30569.2871iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1398 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 215274.0000iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5086 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8243.0215iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:02:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487824 41.610020 41.430065 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842285 68.037109 67.777344 +util 92.250000 94.000000 89.000000 +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 651 : avg iteration time 47.89 seconds +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 651 : data loading time 0.00 +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 651 : avg forward pass time 13.02 +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 651 : backward pass time 10.15 +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 651 : optimizer step time 1.55 +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 651-------------------- +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:03:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488029 41.610836 41.430065 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.955078 67.960938 67.953125 +util 92.250000 100.000000 88.000000 +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 652 : avg iteration time 46.16 seconds +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 652 : data loading time 0.00 +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 652 : avg forward pass time 12.14 +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 652 : backward pass time 10.14 +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 652 : optimizer step time 1.59 +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 652-------------------- +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:04:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488094 41.611095 41.430065 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.940430 67.960938 67.933594 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 653 : avg iteration time 48.54 seconds +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 653 : data loading time 0.00 +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 653 : avg forward pass time 13.34 +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 653 : backward pass time 10.14 +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 653 : optimizer step time 1.56 +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 653-------------------- +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:05:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488104 41.611095 41.430077 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.720703 67.960938 67.640625 +util 99.250000 100.000000 98.000000 +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 654 : avg iteration time 48.61 seconds +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 654 : data loading time 0.00 +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 654 : avg forward pass time 13.37 +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 654 : backward pass time 10.14 +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 654 : optimizer step time 1.56 +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 654-------------------- +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:05:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487651 41.609287 41.430084 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.983887 136.076172 135.953125 +util 64.000000 72.000000 57.000000 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 655 : avg iteration time 160.04 seconds +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 655 : data loading time 0.00 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 655 : avg forward pass time 71.44 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 655 : backward pass time 8.41 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 655 : optimizer step time 0.40 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 655-------------------- +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 289838.9179 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4579 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 289838.9179iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6679iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5227 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3312 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 687670.8750iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 21.0169iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 17233750.0000iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4579iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0206 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3106.3584iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.0094 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 165833.3750iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7290 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1733.8198iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.0641 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 185016.7188iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487643 41.609287 41.430069 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500488 68.017578 67.328125 +util 93.250000 95.000000 90.000000 +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 656 : avg iteration time 50.07 seconds +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 656 : data loading time 0.00 +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 656 : avg forward pass time 14.11 +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 656 : backward pass time 10.13 +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 656 : optimizer step time 1.58 +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 656-------------------- +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:09:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488040 41.610722 41.430069 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.486328 68.019531 67.308594 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 657 : avg iteration time 45.85 seconds +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 657 : data loading time 0.00 +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 657 : avg forward pass time 12.02 +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 657 : backward pass time 10.10 +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 657 : optimizer step time 1.56 +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 657-------------------- +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:10:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488301 41.611710 41.430069 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.442383 67.843750 67.308594 +util 94.000000 96.000000 91.000000 +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 658 : avg iteration time 48.56 seconds +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 658 : data loading time 0.00 +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 658 : avg forward pass time 13.37 +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 658 : backward pass time 10.13 +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 658 : optimizer step time 1.56 +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 658-------------------- +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:10:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488324 41.611710 41.430069 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.496094 68.058594 67.308594 +util 94.250000 100.000000 89.000000 +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 659 : avg iteration time 45.92 seconds +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 659 : data loading time 0.00 +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 659 : avg forward pass time 12.02 +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 659 : backward pass time 10.15 +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 659 : optimizer step time 1.58 +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 659-------------------- +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:11:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487727 41.609356 41.430073 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.042480 136.134766 136.011719 +util 70.250000 72.000000 67.000000 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 660 : avg iteration time 159.57 seconds +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 660 : data loading time 0.00 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 660 : avg forward pass time 71.19 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 660 : backward pass time 8.41 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 660 : optimizer step time 0.40 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 660-------------------- +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 118758.7043 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4814 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 118758.7043iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.8849iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7492 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5298 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 244173.4531iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.4749 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3469915.7500iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4814iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4235 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 6954.0024iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1653 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 226517.5938iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.9842 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 52.8974 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6464 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10858.8535iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:14:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487856 41.609829 41.430073 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.480957 67.822266 67.367188 +util 99.250000 100.000000 99.000000 +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 661 : avg iteration time 47.40 seconds +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 661 : data loading time 0.00 +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 661 : avg forward pass time 12.79 +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 661 : backward pass time 10.13 +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 661 : optimizer step time 1.56 +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 661-------------------- +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:15:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488361 41.611797 41.430073 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.461914 67.863281 67.328125 +util 91.750000 99.000000 88.000000 +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 662 : avg iteration time 48.44 seconds +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 662 : data loading time 0.00 +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 662 : avg forward pass time 13.30 +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 662 : backward pass time 10.13 +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 662 : optimizer step time 1.58 +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 662-------------------- +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:15:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488344 41.611797 41.430073 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.505859 68.039062 67.328125 +util 91.750000 94.000000 89.000000 +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 663 : avg iteration time 43.24 seconds +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 663 : data loading time 0.00 +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 663 : avg forward pass time 10.69 +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 663 : backward pass time 10.14 +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 663 : optimizer step time 1.57 +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 663-------------------- +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488388 41.611797 41.430073 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500977 68.019531 67.328125 +util 93.750000 99.000000 88.000000 +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 664 : avg iteration time 45.94 seconds +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 664 : data loading time 0.00 +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 664 : avg forward pass time 12.05 +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 664 : backward pass time 10.15 +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 664 : optimizer step time 1.56 +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 664-------------------- +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:17:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487878 41.609772 41.430092 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.032715 136.095703 136.011719 +util 66.250000 72.000000 56.000000 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 665 : avg iteration time 167.59 seconds +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 665 : data loading time 0.00 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 665 : avg forward pass time 75.20 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 665 : backward pass time 8.41 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 665 : optimizer step time 0.40 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 665-------------------- +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 153662.1631 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5251 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 153662.1631iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9396iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.7725 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5050 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 302786.8125iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 34.6363iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3302042.2500iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5251iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.5256 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 465633.9688iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6035 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 73639.0078iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8734 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2314.2510iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.5365 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 159.6556iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:20:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487920 41.609787 41.430092 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.510254 68.056641 67.328125 +util 93.750000 99.000000 88.000000 +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 666 : avg iteration time 44.80 seconds +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 666 : data loading time 0.00 +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 666 : avg forward pass time 11.47 +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 666 : backward pass time 10.13 +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 666 : optimizer step time 1.57 +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 666-------------------- +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:20:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488235 41.611046 41.430096 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.608398 68.039062 67.464844 +util 91.250000 93.000000 88.000000 +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 667 : avg iteration time 45.86 seconds +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 667 : data loading time 0.00 +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 667 : avg forward pass time 12.00 +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 667 : backward pass time 10.15 +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 667 : optimizer step time 1.57 +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 667-------------------- +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:21:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488231 41.611046 41.430092 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.608398 68.039062 67.464844 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 668 : avg iteration time 45.88 seconds +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 668 : data loading time 0.00 +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 668 : avg forward pass time 12.01 +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 668 : backward pass time 10.15 +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 668 : optimizer step time 1.57 +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 668-------------------- +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:22:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488234 41.611046 41.430092 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.608398 68.039062 67.464844 +util 93.750000 96.000000 91.000000 +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 669 : avg iteration time 48.55 seconds +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 669 : data loading time 0.00 +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 669 : avg forward pass time 13.35 +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 669 : backward pass time 10.14 +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 669 : optimizer step time 1.57 +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 669-------------------- +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:23:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.487998 41.610039 41.430130 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.032715 136.095703 136.011719 +util 65.000000 73.000000 58.000000 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 670 : avg iteration time 164.94 seconds +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 670 : data loading time 0.00 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 670 : avg forward pass time 73.87 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 670 : backward pass time 8.44 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 670 : optimizer step time 0.40 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 670-------------------- +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 2918124.3269 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5121 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 2918124.3269iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6089iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0813 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.3694 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 4922093.5000iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.6945 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 137458224.0000iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5121iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 9.3694 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 137458224.0000iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5009 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 59983.6445iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.0594 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 24805.7715iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.2162 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 4593.4575iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:26:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488059 41.610275 41.430149 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.407715 67.822266 67.269531 +util 93.250000 95.000000 90.000000 +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 671 : avg iteration time 45.06 seconds +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 671 : data loading time 0.00 +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 671 : avg forward pass time 11.60 +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 671 : backward pass time 10.13 +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 671 : optimizer step time 1.58 +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 671-------------------- +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0164 iter count: 1.0 +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:26:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488549 41.612206 41.430149 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.476562 68.039062 67.289062 +util 98.250000 100.000000 96.000000 +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 672 : avg iteration time 45.87 seconds +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 672 : data loading time 0.00 +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 672 : avg forward pass time 12.01 +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 672 : backward pass time 10.12 +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 672 : optimizer step time 1.57 +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 672-------------------- +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:27:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488626 41.612492 41.430153 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471680 68.019531 67.289062 +util 91.750000 99.000000 88.000000 +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 673 : avg iteration time 43.28 seconds +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 673 : data loading time 0.00 +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 673 : avg forward pass time 10.73 +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 673 : backward pass time 10.12 +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 673 : optimizer step time 1.57 +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 673-------------------- +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:28:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488722 41.612885 41.430161 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.466797 68.058594 67.269531 +util 91.000000 93.000000 88.000000 +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 674 : avg iteration time 43.23 seconds +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 674 : data loading time 0.00 +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 674 : avg forward pass time 10.70 +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 674 : backward pass time 10.13 +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 674 : optimizer step time 1.57 +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 674-------------------- +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:28:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488344 41.611267 41.430241 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.076660 136.095703 136.070312 +util 64.250000 72.000000 57.000000 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 675 : avg iteration time 167.56 seconds +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 675 : data loading time 0.00 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 675 : avg forward pass time 75.18 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 675 : backward pass time 8.43 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 675 : optimizer step time 0.40 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 675-------------------- +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 770284.9531 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5129 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 770284.9531iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1883iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7457 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.7054 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1916436.5000iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.4437 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 36425304.0000iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5129iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3756 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 46687.7852iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6213 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 76310.4531iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.4370 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 390003.8125iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.9949 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2950.8855iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:31:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488345 41.611267 41.430237 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.383301 68.017578 67.171875 +util 94.750000 97.000000 92.000000 +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 676 : avg iteration time 47.49 seconds +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 676 : data loading time 0.00 +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 676 : avg forward pass time 12.80 +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 676 : backward pass time 10.14 +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 676 : optimizer step time 1.57 +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 676-------------------- +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0143 iter count: 1.0 +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:32:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488585 41.612236 41.430237 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.764648 68.019531 67.679688 +util 94.250000 96.000000 91.000000 +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 677 : avg iteration time 48.59 seconds +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 677 : data loading time 0.00 +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 677 : avg forward pass time 13.37 +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 677 : backward pass time 10.13 +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 677 : optimizer step time 1.57 +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 677-------------------- +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:33:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488596 41.612244 41.430237 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.779297 68.078125 67.679688 +util 99.500000 100.000000 99.000000 +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 678 : avg iteration time 48.49 seconds +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 678 : data loading time 0.00 +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 678 : avg forward pass time 13.31 +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 678 : backward pass time 10.16 +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 678 : optimizer step time 1.56 +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 678-------------------- +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:34:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488659 41.612492 41.430256 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.774414 68.058594 67.679688 +util 94.250000 100.000000 89.000000 +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 679 : avg iteration time 43.24 seconds +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 679 : data loading time 0.00 +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 679 : avg forward pass time 10.71 +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 679 : backward pass time 10.12 +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 679 : optimizer step time 1.57 +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 679-------------------- +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:34:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488493 41.611240 41.430355 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.979004 136.056641 135.953125 +util 68.250000 70.000000 65.000000 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 680 : avg iteration time 164.56 seconds +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 680 : data loading time 0.00 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 680 : avg forward pass time 73.68 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 680 : backward pass time 8.41 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 680 : optimizer step time 0.40 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 680-------------------- +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 79812.8223 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5640 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 79812.8223iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4978iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4001 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.8708 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 143224.0938iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.2259 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 928756.5625iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5640iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8448 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2185.3584iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.3232 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5689.8413iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.9742 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 51.8548 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6272 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10451.1074iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:37:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488610 41.611717 41.430378 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.446777 68.037109 67.250000 +util 99.000000 100.000000 98.000000 +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 681 : avg iteration time 44.80 seconds +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 681 : data loading time 0.00 +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 681 : avg forward pass time 11.48 +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 681 : backward pass time 10.14 +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 681 : optimizer step time 1.57 +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 681-------------------- +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:38:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:39:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488870 41.612755 41.430378 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471680 68.019531 67.289062 +util 96.250000 98.000000 93.000000 +[May 5, 2026 - 08:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 682 : avg iteration time 45.96 seconds +[May 5, 2026 - 08:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 682 : data loading time 0.00 +[May 5, 2026 - 08:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 682 : avg forward pass time 12.04 +[May 5, 2026 - 08:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 682 : backward pass time 10.15 +[May 5, 2026 - 08:39:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 682 : optimizer step time 1.56 +[May 5, 2026 - 08:39:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 682-------------------- +[May 5, 2026 - 08:39:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 08:39:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 08:39:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:39:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488929 41.613007 41.430355 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.457031 68.078125 67.250000 +util 99.250000 100.000000 98.000000 +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 683 : avg iteration time 43.25 seconds +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 683 : data loading time 0.00 +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 683 : avg forward pass time 10.70 +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 683 : backward pass time 10.14 +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 683 : optimizer step time 1.56 +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 683-------------------- +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:39:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489007 41.613285 41.430397 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471680 68.078125 67.269531 +util 98.250000 99.000000 96.000000 +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 684 : avg iteration time 48.58 seconds +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 684 : data loading time 0.00 +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 684 : avg forward pass time 13.36 +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 684 : backward pass time 10.14 +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 684 : optimizer step time 1.58 +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 684-------------------- +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:40:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488445 41.611034 41.430378 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.998535 136.134766 135.953125 +util 67.250000 70.000000 64.000000 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 685 : avg iteration time 162.40 seconds +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 685 : data loading time 0.00 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 685 : avg forward pass time 72.62 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 685 : backward pass time 8.41 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 685 : optimizer step time 0.40 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 685-------------------- +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 911707.6250 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5106 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 911707.6250iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7259iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9681 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.9989 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2330334.2500iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.9324 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 65510108.0000iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5106iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.8393 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 118018.0312iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.3818 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 865.8123iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.5326 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 8649.8076iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.2517 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 667.3731iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:43:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488446 41.611034 41.430382 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.905762 67.933594 67.822266 +util 97.250000 99.000000 94.000000 +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 686 : avg iteration time 47.39 seconds +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 686 : data loading time 0.00 +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 686 : avg forward pass time 12.78 +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 686 : backward pass time 10.12 +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 686 : optimizer step time 1.57 +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 686-------------------- +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:44:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488810 41.612507 41.430363 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.950195 68.000000 67.933594 +util 91.500000 99.000000 88.000000 +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 687 : avg iteration time 45.89 seconds +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 687 : data loading time 0.00 +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 687 : avg forward pass time 12.02 +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 687 : backward pass time 10.13 +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 687 : optimizer step time 1.58 +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 687-------------------- +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:44:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488811 41.612495 41.430363 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.974609 68.039062 67.953125 +util 93.750000 99.000000 88.000000 +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 688 : avg iteration time 43.19 seconds +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 688 : data loading time 0.00 +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 688 : avg forward pass time 10.68 +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 688 : backward pass time 10.14 +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 688 : optimizer step time 1.56 +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 688-------------------- +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:45:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488807 41.612495 41.430363 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.974609 68.039062 67.953125 +util 98.250000 99.000000 96.000000 +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 689 : avg iteration time 45.94 seconds +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 689 : data loading time 0.00 +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 689 : avg forward pass time 12.04 +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 689 : backward pass time 10.15 +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 689 : optimizer step time 1.56 +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 689-------------------- +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:46:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488445 41.611038 41.430363 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.006836 136.056641 135.990234 +util 62.250000 64.000000 59.000000 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 690 : avg iteration time 167.74 seconds +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 690 : data loading time 0.00 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 690 : avg forward pass time 75.30 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 690 : backward pass time 8.38 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 690 : optimizer step time 0.40 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 690-------------------- +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 235778.1279 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5328 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 235778.1279iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7718iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4491 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6703 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 391873.2500iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 18.1422iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4595182.0000iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5328iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7061 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 90425.5391iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.3319 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5789.8198iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4362 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 965.2811iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.3229 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 42013.4180iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:49:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488452 41.611042 41.430386 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.505371 67.978516 67.347656 +util 95.750000 98.000000 93.000000 +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 691 : avg iteration time 50.03 seconds +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 691 : data loading time 0.00 +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 691 : avg forward pass time 14.08 +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 691 : backward pass time 10.14 +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 691 : optimizer step time 1.56 +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 691-------------------- +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0144 iter count: 1.0 +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0144 iter count: 1.0 +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:50:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488823 41.612518 41.430363 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.588867 67.960938 67.464844 +util 96.500000 100.000000 88.000000 +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 692 : avg iteration time 48.48 seconds +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 692 : data loading time 0.00 +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 692 : avg forward pass time 13.32 +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 692 : backward pass time 10.12 +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 692 : optimizer step time 1.57 +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 692-------------------- +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:50:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488836 41.612541 41.430370 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.261719 68.000000 67.015625 +util 93.000000 95.000000 90.000000 +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 693 : avg iteration time 45.89 seconds +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 693 : data loading time 0.00 +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 693 : avg forward pass time 12.02 +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 693 : backward pass time 10.14 +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 693 : optimizer step time 1.56 +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 693-------------------- +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:51:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488843 41.612568 41.430370 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.598633 68.000000 67.464844 +util 96.500000 99.000000 93.000000 +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 694 : avg iteration time 45.85 seconds +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 694 : data loading time 0.00 +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 694 : avg forward pass time 12.00 +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 694 : backward pass time 10.12 +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 694 : optimizer step time 1.58 +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 694-------------------- +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:52:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488368 41.610622 41.430405 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.022949 136.056641 136.011719 +util 62.250000 65.000000 59.000000 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 695 : avg iteration time 164.97 seconds +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 695 : data loading time 0.00 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 695 : avg forward pass time 73.87 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 695 : backward pass time 8.43 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 695 : optimizer step time 0.40 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 695-------------------- +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 443519.1680 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5450 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 443519.1680iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8743iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7420 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0787 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 700924.5000iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.4103 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10401209.0000iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5450iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.5725 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1267.6818iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.2048 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 607.6501iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.8811 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 948091.2500iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 8.0787 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10401209.0000iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:55:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488424 41.610847 41.430405 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.490723 67.978516 67.328125 +util 94.500000 97.000000 91.000000 +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 696 : avg iteration time 50.01 seconds +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 696 : data loading time 0.00 +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 696 : avg forward pass time 14.08 +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 696 : backward pass time 10.13 +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 696 : optimizer step time 1.58 +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 696-------------------- +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:55:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488983 41.613060 41.430405 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.339844 68.019531 67.113281 +util 93.000000 95.000000 90.000000 +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 697 : avg iteration time 43.38 seconds +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 697 : data loading time 0.00 +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 697 : avg forward pass time 10.77 +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 697 : backward pass time 10.14 +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 697 : optimizer step time 1.57 +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 697-------------------- +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:56:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489042 41.613304 41.430405 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.574219 68.019531 67.425781 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 698 : avg iteration time 45.89 seconds +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 698 : data loading time 0.00 +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 698 : avg forward pass time 12.02 +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 698 : backward pass time 10.14 +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 698 : optimizer step time 1.57 +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 698-------------------- +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:57:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489041 41.613304 41.430416 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.251953 68.019531 66.996094 +util 90.500000 93.000000 87.000000 +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 699 : avg iteration time 45.89 seconds +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 699 : data loading time 0.00 +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 699 : avg forward pass time 12.01 +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 699 : backward pass time 10.13 +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 699 : optimizer step time 1.57 +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 699-------------------- +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 08:58:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.488555 41.611359 41.430416 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.003418 136.037109 135.992188 +util 69.000000 71.000000 66.000000 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 700 : avg iteration time 162.33 seconds +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 700 : data loading time 0.00 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 700 : avg forward pass time 72.55 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 700 : backward pass time 8.44 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 700 : optimizer step time 0.41 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 700-------------------- +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 49796.4125 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4523 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 49796.4125iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5887iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5171 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2948 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 112610.5391iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.8128 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2168543.2500iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4523iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1968 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 598.0195iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2342 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4761.8677iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7931 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1970.7665iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6741 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1553.3650iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 09:00:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 09:01:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.69 seconds +[May 5, 2026 - 09:01:11 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 700 +[May 5, 2026 - 09:01:11 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000700 +[May 5, 2026 - 09:01:26 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 700 +[May 5, 2026 - 09:01:27 | INFO | fastgen.trainer:validate:358 ] Validation iteration 700 +[May 5, 2026 - 09:01:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:01:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 5, 2026 - 09:01:32 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.645 +[May 5, 2026 - 09:01:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:01:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 09:01:37 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.262 +[May 5, 2026 - 09:01:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:01:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 09:01:42 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.316 +[May 5, 2026 - 09:01:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:01:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 09:01:47 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.439 +[May 5, 2026 - 09:01:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:01:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 09:01:52 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.250 +[May 5, 2026 - 09:01:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:01:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 09:01:57 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=5.901 +[May 5, 2026 - 09:01:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:01:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 09:02:02 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.256 +[May 5, 2026 - 09:02:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:02:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 09:02:07 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.577 +[May 5, 2026 - 09:02:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:02:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 5, 2026 - 09:02:12 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.920 +[May 5, 2026 - 09:02:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 09:02:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 09:02:17 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.387 +[May 5, 2026 - 09:02:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 700-------------------- +[May 5, 2026 - 09:02:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 09:02:33 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 700 +[May 5, 2026 - 09:02:33 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.195 (n=10) +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490541 41.619473 41.430161 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.155273 68.585938 68.011719 +util 93.500000 99.000000 87.000000 +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 701 : avg iteration time 141.83 seconds +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 701 : data loading time 0.00 +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 701 : avg forward pass time 11.70 +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 701 : backward pass time 10.22 +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 701 : optimizer step time 1.57 +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 701-------------------- +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:03:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490101 41.617760 41.430161 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.125977 68.761719 67.914062 +util 90.750000 93.000000 88.000000 +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 702 : avg iteration time 48.65 seconds +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 702 : data loading time 0.00 +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 702 : avg forward pass time 13.37 +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 702 : backward pass time 10.17 +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 702 : optimizer step time 1.57 +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 702-------------------- +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:04:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490110 41.617760 41.430161 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.165039 68.566406 68.031250 +util 96.250000 99.000000 93.000000 +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 703 : avg iteration time 48.67 seconds +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 703 : data loading time 0.00 +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 703 : avg forward pass time 13.38 +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 703 : backward pass time 10.16 +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 703 : optimizer step time 1.57 +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 703-------------------- +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:04:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490116 41.617783 41.430161 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.218750 68.781250 68.031250 +util 96.000000 98.000000 93.000000 +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 704 : avg iteration time 43.39 seconds +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 704 : data loading time 0.00 +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 704 : avg forward pass time 10.76 +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 704 : backward pass time 10.15 +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 704 : optimizer step time 1.56 +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 704-------------------- +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:05:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489662 41.614494 41.430611 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.052246 136.056641 136.050781 +util 67.000000 72.000000 57.000000 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 705 : avg iteration time 162.47 seconds +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 705 : data loading time 0.00 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 705 : avg forward pass time 72.65 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 705 : backward pass time 8.39 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 705 : optimizer step time 0.40 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 705-------------------- +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 115080.3228 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4584 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 115080.3228iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1888iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4592 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3570 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 211132.9219iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5055 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2455756.5000iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4584iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.1994 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 242499.7656iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.9966 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 54.2236 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4716 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 7655.5181iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.8411 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 118450.0000iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:08:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489667 41.614494 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.033203 68.214844 67.972656 +util 89.500000 92.000000 87.000000 +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 706 : avg iteration time 50.24 seconds +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 706 : data loading time 0.00 +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 706 : avg forward pass time 14.18 +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 706 : backward pass time 10.15 +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 706 : optimizer step time 1.59 +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 706-------------------- +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0166 iter count: 1.0 +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:09:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489979 41.615742 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.828125 68.039062 67.757812 +util 91.250000 94.000000 88.000000 +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 707 : avg iteration time 48.61 seconds +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 707 : data loading time 0.00 +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 707 : avg forward pass time 13.38 +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 707 : backward pass time 10.16 +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 707 : optimizer step time 1.56 +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 707-------------------- +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:10:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489966 41.615696 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.038086 68.058594 68.031250 +util 97.250000 100.000000 94.000000 +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 708 : avg iteration time 45.87 seconds +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 708 : data loading time 0.00 +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 708 : avg forward pass time 12.00 +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 708 : backward pass time 10.15 +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 708 : optimizer step time 1.58 +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 708-------------------- +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:10:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490033 41.615963 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.886719 68.214844 67.777344 +util 96.750000 100.000000 88.000000 +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 709 : avg iteration time 43.26 seconds +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 709 : data loading time 0.00 +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 709 : avg forward pass time 10.70 +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 709 : backward pass time 10.13 +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 709 : optimizer step time 1.58 +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 709-------------------- +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:11:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489569 41.614105 41.430611 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.120605 136.128906 136.095703 +util 65.500000 74.000000 58.000000 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 710 : avg iteration time 162.23 seconds +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 710 : data loading time 0.00 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 710 : avg forward pass time 72.52 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 710 : backward pass time 8.40 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 710 : optimizer step time 0.41 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 710-------------------- +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 13821979.8379 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4762 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 13821979.8379iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.2197iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4508 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 10.3111iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 28587068.0000iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.4634 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 903839040.0000iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4762iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.4440 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 53529.0352iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.6463 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 10857.9844iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.9000 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 133247.0312iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.9506 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 365.4714iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:14:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489557 41.614059 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.983887 68.109375 67.607422 +util 91.250000 94.000000 88.000000 +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 711 : avg iteration time 44.83 seconds +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 711 : data loading time 0.00 +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 711 : avg forward pass time 11.48 +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 711 : backward pass time 10.14 +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 711 : optimizer step time 1.59 +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 711-------------------- +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:14:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489737 41.614777 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.866699 67.933594 67.666016 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 712 : avg iteration time 48.47 seconds +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 712 : data loading time 0.00 +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 712 : avg forward pass time 13.31 +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 712 : backward pass time 10.14 +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 712 : optimizer step time 1.56 +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 712-------------------- +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:15:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489775 41.614777 41.430618 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.876465 67.953125 67.646484 +util 89.250000 92.000000 86.000000 +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 713 : avg iteration time 45.90 seconds +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 713 : data loading time 0.00 +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 713 : avg forward pass time 12.03 +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 713 : backward pass time 10.15 +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 713 : optimizer step time 1.57 +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 713-------------------- +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:16:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489770 41.614777 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852051 67.933594 67.607422 +util 99.250000 100.000000 98.000000 +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 714 : avg iteration time 45.93 seconds +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 714 : data loading time 0.00 +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 714 : avg forward pass time 12.05 +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 714 : backward pass time 10.12 +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 714 : optimizer step time 1.58 +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 714-------------------- +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:17:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489397 41.613277 41.430611 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.008301 136.011719 135.998047 +util 61.250000 64.000000 58.000000 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 715 : avg iteration time 165.10 seconds +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 715 : data loading time 0.00 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 715 : avg forward pass time 73.94 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 715 : backward pass time 8.42 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 715 : optimizer step time 0.40 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 715-------------------- +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 493883.5464 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4913 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 493883.5464iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0356iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1220 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6419 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1168416.3750iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.4308 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 32078824.0000iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4913iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.1700 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 228661.5625iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4482 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 53984.8984iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.8560 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 40.9362 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.6872 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 87065.6953iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:20:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489457 41.613522 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.812988 67.894531 67.568359 +util 94.250000 97.000000 91.000000 +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 716 : avg iteration time 47.42 seconds +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 716 : data loading time 0.00 +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 716 : avg forward pass time 12.79 +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 716 : backward pass time 10.13 +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 716 : optimizer step time 1.58 +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 716-------------------- +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0144 iter count: 1.0 +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0144 iter count: 1.0 +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:20:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489712 41.614517 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.686035 67.718750 67.587891 +util 98.000000 100.000000 95.000000 +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 717 : avg iteration time 45.84 seconds +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 717 : data loading time 0.00 +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 717 : avg forward pass time 12.00 +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 717 : backward pass time 10.14 +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 717 : optimizer step time 1.55 +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 717-------------------- +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:21:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489705 41.614513 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.681152 67.718750 67.568359 +util 96.500000 99.000000 89.000000 +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 718 : avg iteration time 45.86 seconds +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 718 : data loading time 0.00 +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 718 : avg forward pass time 12.01 +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 718 : backward pass time 10.14 +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 718 : optimizer step time 1.56 +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 718-------------------- +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:22:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489771 41.614777 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.646973 67.718750 67.431641 +util 89.500000 92.000000 87.000000 +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 719 : avg iteration time 43.44 seconds +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 719 : data loading time 0.00 +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 719 : avg forward pass time 10.79 +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 719 : backward pass time 10.14 +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 719 : optimizer step time 1.57 +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 719-------------------- +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:23:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489470 41.613537 41.430611 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.042480 136.050781 136.017578 +util 62.000000 75.000000 53.000000 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 720 : avg iteration time 164.99 seconds +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 720 : data loading time 0.00 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 720 : avg forward pass time 73.91 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 720 : backward pass time 8.42 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 720 : optimizer step time 0.40 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 720-------------------- +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 146544.0615 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4980 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 146544.0615iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8571iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5173 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6007 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 300559.7188iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 20.7925iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3998315.0000iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4980iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.5128 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 8313.7354iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.6007 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3998315.0000iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.7051 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 223.6572iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2478 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 267185.1250iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:25:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489470 41.613533 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842285 67.894531 67.685547 +util 96.500000 100.000000 88.000000 +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 721 : avg iteration time 47.36 seconds +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 721 : data loading time 0.00 +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 721 : avg forward pass time 12.76 +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 721 : backward pass time 10.13 +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 721 : optimizer step time 1.57 +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 721-------------------- +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:26:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489769 41.614746 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.700684 67.718750 67.646484 +util 93.250000 96.000000 90.000000 +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 722 : avg iteration time 48.49 seconds +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 722 : data loading time 0.00 +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 722 : avg forward pass time 13.34 +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 722 : backward pass time 10.12 +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 722 : optimizer step time 1.58 +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 722-------------------- +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:27:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489773 41.614746 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.822754 67.875000 67.666016 +util 96.750000 100.000000 89.000000 +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 723 : avg iteration time 45.92 seconds +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 723 : data loading time 0.00 +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 723 : avg forward pass time 12.03 +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 723 : backward pass time 10.14 +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 723 : optimizer step time 1.57 +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 723-------------------- +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:28:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489773 41.614746 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.720215 67.738281 67.666016 +util 98.250000 99.000000 96.000000 +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 724 : avg iteration time 48.55 seconds +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 724 : data loading time 0.00 +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 724 : avg forward pass time 13.33 +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 724 : backward pass time 10.15 +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 724 : optimizer step time 1.59 +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 724-------------------- +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:29:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489635 41.614159 41.430611 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.008301 136.056641 135.992188 +util 69.250000 72.000000 66.000000 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 725 : avg iteration time 165.02 seconds +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 725 : data loading time 0.00 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 725 : avg forward pass time 73.90 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 725 : backward pass time 8.44 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 725 : optimizer step time 0.40 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 725-------------------- +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 250653.0898 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5164 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 250653.0898iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0193iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.8988 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8446 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 510913.5625iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 44.5902iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6512275.5000iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5164iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.1174 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 69.0430 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3843 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 47510.0156iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1004 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 3644.1282iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1164 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 27799.2754iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:31:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489639 41.614166 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.695801 67.738281 67.568359 +util 96.500000 99.000000 89.000000 +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 726 : avg iteration time 47.43 seconds +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 726 : data loading time 0.00 +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 726 : avg forward pass time 12.79 +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 726 : backward pass time 10.14 +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 726 : optimizer step time 1.57 +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 726-------------------- +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:32:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489697 41.614395 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.803223 67.875000 67.587891 +util 94.000000 99.000000 88.000000 +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 727 : avg iteration time 43.40 seconds +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 727 : data loading time 0.00 +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 727 : avg forward pass time 10.77 +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 727 : backward pass time 10.14 +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 727 : optimizer step time 1.57 +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 727-------------------- +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:33:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489788 41.614662 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.793457 67.875000 67.548828 +util 90.750000 93.000000 88.000000 +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 728 : avg iteration time 48.45 seconds +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 728 : data loading time 0.00 +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 728 : avg forward pass time 13.30 +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 728 : backward pass time 10.13 +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 728 : optimizer step time 1.56 +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 728-------------------- +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:34:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489796 41.614662 41.430618 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.759277 67.875000 67.412109 +util 96.750000 99.000000 94.000000 +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 729 : avg iteration time 45.87 seconds +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 729 : data loading time 0.00 +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 729 : avg forward pass time 12.00 +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 729 : backward pass time 10.13 +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 729 : optimizer step time 1.59 +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 729-------------------- +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:34:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489236 41.612427 41.430618 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.047363 136.095703 136.031250 +util 68.000000 70.000000 65.000000 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 730 : avg iteration time 167.64 seconds +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 730 : data loading time 0.00 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 730 : avg forward pass time 75.22 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 730 : backward pass time 8.39 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 730 : optimizer step time 0.40 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 730-------------------- +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 123842.5693 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4614 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 123842.5693iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4208iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3285 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8583 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 259988.6562iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 14.2536iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6693677.0000iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4614iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0770 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3477.1123iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.0939 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 196360.5781iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7869 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1946.7107iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9100 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 18399.5195iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:37:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489359 41.612926 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.856934 67.933594 67.626953 +util 89.250000 92.000000 86.000000 +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 731 : avg iteration time 47.48 seconds +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 731 : data loading time 0.00 +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 731 : avg forward pass time 12.80 +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 731 : backward pass time 10.16 +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 731 : optimizer step time 1.57 +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 731-------------------- +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:38:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489684 41.614227 41.430611 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.700684 67.718750 67.646484 +util 90.250000 93.000000 87.000000 +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 732 : avg iteration time 48.61 seconds +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 732 : data loading time 0.00 +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 732 : avg forward pass time 13.37 +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 732 : backward pass time 10.14 +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 732 : optimizer step time 1.58 +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 732-------------------- +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:39:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489758 41.614517 41.430618 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.720215 67.757812 67.607422 +util 98.000000 99.000000 96.000000 +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 733 : avg iteration time 45.90 seconds +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 733 : data loading time 0.00 +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 733 : avg forward pass time 12.03 +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 733 : backward pass time 10.15 +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 733 : optimizer step time 1.57 +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 733-------------------- +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:40:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489854 41.614761 41.430725 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.886230 67.972656 67.626953 +util 96.500000 99.000000 89.000000 +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 734 : avg iteration time 43.25 seconds +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 734 : data loading time 0.00 +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 734 : avg forward pass time 10.70 +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 734 : backward pass time 10.15 +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 734 : optimizer step time 1.56 +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 734-------------------- +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:40:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489433 41.613094 41.430725 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.022949 136.056641 136.011719 +util 61.250000 64.000000 58.000000 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 735 : avg iteration time 165.29 seconds +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 735 : data loading time 0.00 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 735 : avg forward pass time 74.04 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 735 : backward pass time 8.40 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 735 : optimizer step time 0.41 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 735-------------------- +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1618853.0630 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4156 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1618853.0630iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9464iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6330 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.1683 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 3627663.0000iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.5466 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 91933360.0000iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4156iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.1644 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1670944.1250iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4958 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 59378.8594iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.5129 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1125.2416iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.5832 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 175.2848iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:43:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489435 41.613102 41.430725 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.920410 68.031250 67.587891 +util 98.000000 99.000000 96.000000 +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 736 : avg iteration time 50.07 seconds +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 736 : data loading time 0.00 +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 736 : avg forward pass time 14.09 +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 736 : backward pass time 10.16 +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 736 : optimizer step time 1.58 +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 736-------------------- +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:44:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489806 41.614586 41.430725 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.022949 68.031250 67.998047 +util 92.000000 95.000000 89.000000 +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 737 : avg iteration time 45.92 seconds +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 737 : data loading time 0.00 +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 737 : avg forward pass time 12.02 +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 737 : backward pass time 10.14 +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 737 : optimizer step time 1.58 +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 737-------------------- +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:45:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489838 41.614586 41.430725 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.920410 67.933594 67.880859 +util 99.000000 100.000000 98.000000 +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 738 : avg iteration time 48.50 seconds +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 738 : data loading time 0.00 +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 738 : avg forward pass time 13.32 +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 738 : backward pass time 10.14 +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 738 : optimizer step time 1.57 +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 738-------------------- +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:45:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489861 41.614681 41.430725 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.979004 68.056641 67.953125 +util 98.500000 99.000000 97.000000 +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 739 : avg iteration time 48.59 seconds +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 739 : data loading time 0.00 +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 739 : avg forward pass time 13.35 +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 739 : backward pass time 10.15 +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 739 : optimizer step time 1.58 +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 739-------------------- +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:46:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489461 41.612843 41.430725 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.062012 136.070312 136.037109 +util 70.000000 73.000000 67.000000 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 740 : avg iteration time 167.62 seconds +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 740 : data loading time 0.00 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 740 : avg forward pass time 75.22 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 740 : backward pass time 8.40 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 740 : optimizer step time 0.40 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 740-------------------- +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 312040.2119 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5190 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 312040.2119iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9040iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6071 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1382 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 671079.0000iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 24.8837iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 11716183.0000iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5190iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2471 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 36105.7148iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1817 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4287.1138iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.8118 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 15118.4990iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.4342 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 52494.9961iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:49:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489454 41.612820 41.430740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.920898 67.933594 67.882812 +util 92.250000 95.000000 89.000000 +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 741 : avg iteration time 49.90 seconds +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 741 : data loading time 0.00 +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 741 : avg forward pass time 14.04 +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 741 : backward pass time 10.13 +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 741 : optimizer step time 1.55 +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 741-------------------- +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:50:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489829 41.614296 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.916016 68.097656 67.855469 +util 96.250000 100.000000 87.000000 +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 742 : avg iteration time 45.91 seconds +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 742 : data loading time 0.00 +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 742 : avg forward pass time 12.04 +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 742 : backward pass time 10.13 +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 742 : optimizer step time 1.56 +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 742-------------------- +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:51:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489829 41.614296 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.935547 67.941406 67.933594 +util 96.500000 99.000000 89.000000 +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 743 : avg iteration time 45.82 seconds +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 743 : data loading time 0.00 +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 743 : avg forward pass time 12.00 +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 743 : backward pass time 10.12 +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 743 : optimizer step time 1.58 +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 743-------------------- +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:51:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489823 41.614296 41.430740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.950195 67.972656 67.882812 +util 91.500000 100.000000 88.000000 +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 744 : avg iteration time 48.41 seconds +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 744 : data loading time 0.00 +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 744 : avg forward pass time 13.29 +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 744 : backward pass time 10.12 +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 744 : optimizer step time 1.57 +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 744-------------------- +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:52:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489495 41.612865 41.430763 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.071777 136.076172 136.070312 +util 60.750000 64.000000 58.000000 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 745 : avg iteration time 160.06 seconds +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 745 : data loading time 0.00 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 745 : avg forward pass time 71.42 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 745 : backward pass time 8.42 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 745 : optimizer step time 0.40 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 745-------------------- +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 149232.9116 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4925 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 149232.9116iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3161iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1583 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7871 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 265477.7500iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.1412iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5805077.5000iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4925iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.5718 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 171.3306iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.6176 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 10251.8584iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.7871 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 5805077.5000iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.6868 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 215.6398iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:55:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489558 41.613117 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.857422 67.953125 67.570312 +util 96.500000 99.000000 94.000000 +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 746 : avg iteration time 47.45 seconds +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 746 : data loading time 0.00 +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 746 : avg forward pass time 12.80 +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 746 : backward pass time 10.14 +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 746 : optimizer step time 1.57 +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 746-------------------- +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:56:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489870 41.614368 41.430740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.862305 67.953125 67.589844 +util 96.500000 100.000000 87.000000 +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 747 : avg iteration time 48.46 seconds +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 747 : data loading time 0.00 +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 747 : avg forward pass time 13.31 +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 747 : backward pass time 10.14 +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 747 : optimizer step time 1.57 +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 747-------------------- +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:56:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489868 41.614368 41.430740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.818359 67.914062 67.531250 +util 91.250000 94.000000 88.000000 +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 748 : avg iteration time 48.49 seconds +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 748 : data loading time 0.00 +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 748 : avg forward pass time 13.31 +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 748 : backward pass time 10.16 +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 748 : optimizer step time 1.56 +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 748-------------------- +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:57:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489874 41.614380 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.671875 67.757812 67.414062 +util 92.000000 95.000000 89.000000 +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 749 : avg iteration time 46.00 seconds +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 749 : data loading time 0.00 +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 749 : avg forward pass time 12.07 +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 749 : backward pass time 10.15 +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 749 : optimizer step time 1.58 +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 749-------------------- +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 09:58:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489929 41.614578 41.430763 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.983887 136.076172 135.953125 +util 60.500000 63.000000 57.000000 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 750 : avg iteration time 165.00 seconds +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 750 : data loading time 0.00 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 750 : avg forward pass time 73.91 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 750 : backward pass time 8.40 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 750 : optimizer step time 0.40 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 750-------------------- +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 584789.3389 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4485 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 584789.3389iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4534iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9074 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3816 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 802535.6250iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.1394 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 19062746.0000iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4485iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.4190 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 932.6819iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.9379 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2632.6489iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2244 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4669.0254iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.5135 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 61513.3828iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:01:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489934 41.614601 41.430740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.793457 68.076172 67.699219 +util 91.750000 94.000000 89.000000 +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 751 : avg iteration time 47.40 seconds +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 751 : data loading time 0.00 +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 751 : avg forward pass time 12.77 +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 751 : backward pass time 10.14 +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 751 : optimizer step time 1.58 +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 751-------------------- +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:02:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489924 41.614578 41.430740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.983887 68.076172 67.953125 +util 96.500000 100.000000 88.000000 +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 752 : avg iteration time 45.93 seconds +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 752 : data loading time 0.00 +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 752 : avg forward pass time 12.03 +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 752 : backward pass time 10.14 +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 752 : optimizer step time 1.59 +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 752-------------------- +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:02:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490001 41.614845 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.979004 68.056641 67.953125 +util 97.750000 99.000000 95.000000 +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 753 : avg iteration time 43.35 seconds +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 753 : data loading time 0.00 +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 753 : avg forward pass time 10.73 +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 753 : backward pass time 10.14 +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 753 : optimizer step time 1.58 +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 753-------------------- +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:03:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489992 41.614830 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.022949 68.056641 68.011719 +util 96.750000 99.000000 90.000000 +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 754 : avg iteration time 48.67 seconds +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 754 : data loading time 0.00 +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 754 : avg forward pass time 13.39 +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 754 : backward pass time 10.15 +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 754 : optimizer step time 1.57 +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 754-------------------- +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:04:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489185 41.611599 41.430740 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.076172 136.011719 +util 64.750000 67.000000 62.000000 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 755 : avg iteration time 162.31 seconds +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 755 : data loading time 0.00 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 755 : avg forward pass time 72.56 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 755 : backward pass time 8.44 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 755 : optimizer step time 0.40 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 755-------------------- +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 95091.1218 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4573 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 95091.1218iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4028iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2815 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5528 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 222793.8438iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.9750iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3632763.2500iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4573iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.5528 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3632763.2500iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.7450 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1790.1815iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4350 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 962.9972iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.0168 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 417.2294iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:07:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489183 41.611599 41.430740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.895996 67.900391 67.894531 +util 96.000000 99.000000 93.000000 +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 756 : avg iteration time 47.39 seconds +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 756 : data loading time 0.00 +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 756 : avg forward pass time 12.77 +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 756 : backward pass time 10.14 +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 756 : optimizer step time 1.57 +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 756-------------------- +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0152 iter count: 1.0 +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0152 iter count: 1.0 +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:07:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489564 41.613091 41.430767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.817871 67.894531 67.587891 +util 97.750000 99.000000 95.000000 +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 757 : avg iteration time 43.23 seconds +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 757 : data loading time 0.00 +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 757 : avg forward pass time 10.70 +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 757 : backward pass time 10.12 +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 757 : optimizer step time 1.57 +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 757-------------------- +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:08:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489676 41.613571 41.430744 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.969238 67.972656 67.958984 +util 98.500000 100.000000 96.000000 +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 758 : avg iteration time 45.81 seconds +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 758 : data loading time 0.00 +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 758 : avg forward pass time 11.99 +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 758 : backward pass time 10.12 +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 758 : optimizer step time 1.57 +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 758-------------------- +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:09:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489680 41.613571 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.969238 67.972656 67.958984 +util 96.500000 99.000000 89.000000 +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 759 : avg iteration time 45.85 seconds +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 759 : data loading time 0.00 +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 759 : avg forward pass time 12.01 +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 759 : backward pass time 10.13 +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 759 : optimizer step time 1.57 +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 759-------------------- +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:10:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489215 41.611679 41.430786 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.096191 136.109375 136.056641 +util 60.250000 63.000000 57.000000 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 760 : avg iteration time 164.86 seconds +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 760 : data loading time 0.00 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 760 : avg forward pass time 73.83 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 760 : backward pass time 8.41 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 760 : optimizer step time 0.40 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 760-------------------- +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 274420.2617 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4854 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 274420.2617iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4845iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4024 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1354 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 595493.6875iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.2362 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 11649978.0000iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4854iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7159 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 92205.2422iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 0.4024 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2.2362 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.5083 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1114.9506iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.4114 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 918.5425iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:12:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489272 41.611931 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.910645 67.972656 67.724609 +util 91.000000 94.000000 88.000000 +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 761 : avg iteration time 47.39 seconds +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 761 : data loading time 0.00 +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 761 : avg forward pass time 12.76 +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 761 : backward pass time 10.15 +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 761 : optimizer step time 1.57 +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 761-------------------- +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:13:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489534 41.612957 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.729980 67.738281 67.705078 +util 99.250000 100.000000 97.000000 +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 762 : avg iteration time 45.91 seconds +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 762 : data loading time 0.00 +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 762 : avg forward pass time 12.03 +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 762 : backward pass time 10.12 +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 762 : optimizer step time 1.58 +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 762-------------------- +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:14:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489523 41.612934 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.725098 67.738281 67.685547 +util 91.000000 94.000000 88.000000 +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 763 : avg iteration time 48.49 seconds +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 763 : data loading time 0.00 +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 763 : avg forward pass time 13.31 +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 763 : backward pass time 10.15 +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 763 : optimizer step time 1.57 +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 763-------------------- +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:15:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489594 41.613201 41.430763 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.729980 67.738281 67.705078 +util 95.000000 98.000000 92.000000 +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 764 : avg iteration time 48.45 seconds +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 764 : data loading time 0.00 +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 764 : avg forward pass time 13.29 +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 764 : backward pass time 10.13 +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 764 : optimizer step time 1.57 +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 764-------------------- +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:15:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489247 41.611816 41.430763 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.042480 136.134766 136.011719 +util 66.000000 69.000000 63.000000 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 765 : avg iteration time 164.64 seconds +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 765 : data loading time 0.00 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 765 : avg forward pass time 73.72 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 765 : backward pass time 8.45 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 765 : optimizer step time 0.40 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 765-------------------- +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 426019.8945 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5102 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 426019.8945iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.2904iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.8020 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0323 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 707989.6250iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 271.4958iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9479643.0000iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5102iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.5594 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 498196.8438iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.8258 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2103.8525iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2154 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4586.2788iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.8532 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 16422.5312iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:18:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.24 seconds +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489253 41.611816 41.430786 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.818359 67.875000 67.648438 +util 91.000000 94.000000 88.000000 +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 766 : avg iteration time 44.92 seconds +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 766 : data loading time 0.00 +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 766 : avg forward pass time 11.42 +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 766 : backward pass time 10.14 +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 766 : optimizer step time 1.59 +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 766-------------------- +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:19:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489563 41.613071 41.430767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852539 67.933594 67.609375 +util 90.000000 93.000000 87.000000 +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 767 : avg iteration time 45.99 seconds +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 767 : data loading time 0.00 +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 767 : avg forward pass time 12.06 +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 767 : backward pass time 10.15 +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 767 : optimizer step time 1.56 +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 767-------------------- +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:20:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489958 41.614632 41.430771 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.813477 67.894531 67.570312 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 768 : avg iteration time 48.49 seconds +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 768 : data loading time 0.00 +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 768 : avg forward pass time 13.32 +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 768 : backward pass time 10.14 +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 768 : optimizer step time 1.57 +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 768-------------------- +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:21:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489953 41.614586 41.430782 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.823242 67.953125 67.433594 +util 95.000000 98.000000 92.000000 +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 769 : avg iteration time 48.56 seconds +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 769 : data loading time 0.00 +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 769 : avg forward pass time 13.33 +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 769 : backward pass time 10.16 +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 769 : optimizer step time 1.57 +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 769-------------------- +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:21:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489478 41.612667 41.430767 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.076172 136.011719 +util 65.000000 68.000000 62.000000 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 770 : avg iteration time 167.63 seconds +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 770 : data loading time 0.00 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 770 : avg forward pass time 75.21 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 770 : backward pass time 8.39 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 770 : optimizer step time 0.40 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 770-------------------- +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 442112.3672 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4565 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 442112.3672iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.0236iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5454 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.4018 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 862102.3125iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.9764 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 19848338.0000iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4565iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.0587 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 453.6902iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6384 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 78970.7266iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8692 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2294.8491iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4153 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 6840.8647iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:24:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489489 41.612713 41.430782 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852539 67.933594 67.609375 +util 95.250000 99.000000 92.000000 +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 771 : avg iteration time 49.98 seconds +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 771 : data loading time 0.00 +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 771 : avg forward pass time 14.05 +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 771 : backward pass time 10.13 +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 771 : optimizer step time 1.59 +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 771-------------------- +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:25:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489791 41.613926 41.430767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.681641 67.718750 67.570312 +util 92.000000 100.000000 88.000000 +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 772 : avg iteration time 45.92 seconds +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 772 : data loading time 0.00 +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 772 : avg forward pass time 12.02 +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 772 : backward pass time 10.16 +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 772 : optimizer step time 1.56 +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 772-------------------- +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:26:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489791 41.613926 41.430767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.876953 67.992188 67.531250 +util 91.000000 94.000000 88.000000 +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 773 : avg iteration time 48.43 seconds +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 773 : data loading time 0.00 +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 773 : avg forward pass time 13.30 +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 773 : backward pass time 10.14 +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 773 : optimizer step time 1.56 +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 773-------------------- +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:27:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489803 41.613949 41.430767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842773 67.992188 67.394531 +util 97.750000 99.000000 96.000000 +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 774 : avg iteration time 45.89 seconds +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 774 : data loading time 0.00 +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 774 : avg forward pass time 12.01 +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 774 : backward pass time 10.15 +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 774 : optimizer step time 1.56 +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 774-------------------- +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:27:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489488 41.612633 41.430767 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.101074 136.128906 136.017578 +util 60.000000 63.000000 57.000000 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 775 : avg iteration time 164.72 seconds +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 775 : data loading time 0.00 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 775 : avg forward pass time 73.80 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 775 : backward pass time 8.40 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 775 : optimizer step time 0.40 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 775-------------------- +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 4336483.9087 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4924 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 4336483.9087iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8348iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9529 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.5868 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 7451995.5000iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.7251 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 212310640.0000iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4924iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.5731 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 9378.9316iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.2650 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 37420.2148iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2561 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 36759.4219iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.4443 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 395738.8438iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:30:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489615 41.613136 41.430767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852539 68.136719 67.757812 +util 92.000000 95.000000 89.000000 +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 776 : avg iteration time 47.40 seconds +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 776 : data loading time 0.00 +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 776 : avg forward pass time 12.77 +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 776 : backward pass time 10.15 +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 776 : optimizer step time 1.56 +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 776-------------------- +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0143 iter count: 1.0 +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489805 41.613888 41.430775 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.906250 68.000000 67.875000 +util 98.500000 100.000000 96.000000 +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 777 : avg iteration time 45.89 seconds +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 777 : data loading time 0.00 +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 777 : avg forward pass time 12.01 +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 777 : backward pass time 10.12 +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 777 : optimizer step time 1.57 +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 777-------------------- +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:32:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489807 41.613888 41.430767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.789062 68.000000 67.718750 +util 95.000000 98.000000 92.000000 +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 778 : avg iteration time 48.55 seconds +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 778 : data loading time 0.00 +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 778 : avg forward pass time 13.36 +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 778 : backward pass time 10.14 +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 778 : optimizer step time 1.56 +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 778-------------------- +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:32:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489816 41.613888 41.430767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.925781 68.019531 67.894531 +util 96.750000 100.000000 89.000000 +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 779 : avg iteration time 45.89 seconds +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 779 : data loading time 0.00 +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 779 : avg forward pass time 12.00 +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 779 : backward pass time 10.16 +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 779 : optimizer step time 1.57 +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 779-------------------- +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:33:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489573 41.612331 41.431000 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.988770 136.037109 135.972656 +util 58.750000 62.000000 55.000000 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 780 : avg iteration time 162.86 seconds +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 780 : data loading time 0.00 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 780 : avg forward pass time 72.80 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 780 : backward pass time 8.44 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 780 : optimizer step time 0.41 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 780-------------------- +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 275555.7441 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5280 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 275555.7441iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9396iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9115 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9266 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 557351.3125iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 45.7382iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7673379.5000iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5280iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2704 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 37824.3594iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5261 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 63081.4023iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.9266 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 7673379.5000iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.2172 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 4602.2803iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:36:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489573 41.612331 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.740234 67.777344 67.628906 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 781 : avg iteration time 50.15 seconds +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 781 : data loading time 0.00 +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 781 : avg forward pass time 14.14 +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 781 : backward pass time 10.15 +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 781 : optimizer step time 1.58 +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 781-------------------- +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:37:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489883 41.613583 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852539 67.914062 67.667969 +util 94.500000 98.000000 91.000000 +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 782 : avg iteration time 48.70 seconds +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 782 : data loading time 0.00 +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 782 : avg forward pass time 13.42 +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 782 : backward pass time 10.14 +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 782 : optimizer step time 1.57 +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 782-------------------- +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:38:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489895 41.613583 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.750000 67.816406 67.550781 +util 98.500000 99.000000 97.000000 +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 783 : avg iteration time 46.06 seconds +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 783 : data loading time 0.00 +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 783 : avg forward pass time 12.10 +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 783 : backward pass time 10.13 +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 783 : optimizer step time 1.59 +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 783-------------------- +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:38:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489889 41.613583 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.720703 67.796875 67.492188 +util 96.250000 99.000000 88.000000 +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 784 : avg iteration time 46.03 seconds +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 784 : data loading time 0.00 +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 784 : avg forward pass time 12.09 +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 784 : backward pass time 10.13 +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 784 : optimizer step time 1.57 +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 784-------------------- +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:39:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489531 41.612160 41.431011 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.031250 136.017578 +util 58.500000 62.000000 55.000000 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 785 : avg iteration time 165.46 seconds +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 785 : data loading time 0.00 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 785 : avg forward pass time 74.17 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 785 : backward pass time 8.36 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 785 : optimizer step time 0.40 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 785-------------------- +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 265162.3330 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4539 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 265162.3330iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0095iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.7446 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0007 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 430170.1250iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 242.0826iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8898526.0000iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4539iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2916 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 291612.0000iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1270 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 209820.8281iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8517 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2215.9487iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.3060 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 300133.1250iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:42:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489591 41.612411 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.935547 68.058594 67.894531 +util 98.250000 99.000000 96.000000 +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 786 : avg iteration time 47.70 seconds +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 786 : data loading time 0.00 +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 786 : avg forward pass time 12.92 +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 786 : backward pass time 10.15 +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 786 : optimizer step time 1.56 +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 786-------------------- +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0144 iter count: 1.0 +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0144 iter count: 1.0 +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:43:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489909 41.613663 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.691406 67.718750 67.609375 +util 96.750000 100.000000 88.000000 +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 787 : avg iteration time 48.64 seconds +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 787 : data loading time 0.00 +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 787 : avg forward pass time 13.38 +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 787 : backward pass time 10.13 +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 787 : optimizer step time 1.59 +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 787-------------------- +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:43:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489916 41.613674 41.431011 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.691406 67.718750 67.609375 +util 91.750000 95.000000 89.000000 +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 788 : avg iteration time 46.08 seconds +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 788 : data loading time 0.00 +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 788 : avg forward pass time 12.10 +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 788 : backward pass time 10.15 +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 788 : optimizer step time 1.57 +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 788-------------------- +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:44:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489912 41.613674 41.431019 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.691406 67.718750 67.609375 +util 91.000000 99.000000 87.000000 +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 789 : avg iteration time 43.35 seconds +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 789 : data loading time 0.00 +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 789 : avg forward pass time 10.74 +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 789 : backward pass time 10.14 +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 789 : optimizer step time 1.57 +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 789-------------------- +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:45:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489546 41.612206 41.431011 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.018066 136.095703 135.992188 +util 60.000000 63.000000 57.000000 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 790 : avg iteration time 165.44 seconds +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 790 : data loading time 0.00 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 790 : avg forward pass time 74.10 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 790 : backward pass time 8.43 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 790 : optimizer step time 0.40 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 790-------------------- +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 358126.3428 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4876 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 358126.3428iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6465iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0589 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3368 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 704428.6250iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.3130 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 17426456.0000iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4876iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.2780 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 703.4963iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.0589 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 8.3130 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8322 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2131.0000iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.0903 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3571.0483iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:48:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489613 41.612457 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.779297 67.816406 67.667969 +util 90.750000 94.000000 88.000000 +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 791 : avg iteration time 47.59 seconds +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 791 : data loading time 0.00 +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 791 : avg forward pass time 12.86 +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 791 : backward pass time 10.13 +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 791 : optimizer step time 1.58 +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 791-------------------- +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:49:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489805 41.613232 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.862305 67.953125 67.589844 +util 96.500000 100.000000 88.000000 +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 792 : avg iteration time 48.74 seconds +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 792 : data loading time 0.00 +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 792 : avg forward pass time 13.42 +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 792 : backward pass time 10.13 +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 792 : optimizer step time 1.57 +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 792-------------------- +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:49:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:50:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489800 41.613224 41.431019 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.701172 67.757812 67.531250 +util 96.750000 100.000000 94.000000 +[May 5, 2026 - 10:50:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 793 : avg iteration time 43.34 seconds +[May 5, 2026 - 10:50:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 793 : data loading time 0.00 +[May 5, 2026 - 10:50:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 793 : avg forward pass time 10.74 +[May 5, 2026 - 10:50:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 793 : backward pass time 10.14 +[May 5, 2026 - 10:50:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 793 : optimizer step time 1.57 +[May 5, 2026 - 10:50:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 793-------------------- +[May 5, 2026 - 10:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 10:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 10:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:50:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489855 41.613457 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.891602 67.953125 67.707031 +util 98.250000 99.000000 96.000000 +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 794 : avg iteration time 46.02 seconds +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 794 : data loading time 0.00 +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 794 : avg forward pass time 12.08 +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 794 : backward pass time 10.15 +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 794 : optimizer step time 1.58 +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 794-------------------- +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:51:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489506 41.612000 41.431000 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.115234 136.011719 +util 59.250000 61.000000 57.000000 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 795 : avg iteration time 168.49 seconds +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 795 : data loading time 0.00 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 795 : avg forward pass time 75.63 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 795 : backward pass time 8.45 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 795 : optimizer step time 0.40 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 795-------------------- +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 552655.8120 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5377 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 552655.8120iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0468iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4141 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3399 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 984296.1250iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 16.9165iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 17536688.0000iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5377iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3749 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 46619.9375iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.5424 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 8820.3594iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.6096 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 184.7685iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.6499 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 80812.4766iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:54:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489566 41.612259 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.901367 68.039062 67.855469 +util 92.500000 96.000000 89.000000 +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 796 : avg iteration time 47.64 seconds +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 796 : data loading time 0.00 +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 796 : avg forward pass time 12.89 +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 796 : backward pass time 10.15 +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 796 : optimizer step time 1.56 +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 796-------------------- +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0159 iter count: 1.0 +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:54:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489827 41.613270 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.969727 68.078125 67.933594 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 797 : avg iteration time 45.93 seconds +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 797 : data loading time 0.00 +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 797 : avg forward pass time 12.03 +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 797 : backward pass time 10.15 +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 797 : optimizer step time 1.57 +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 797-------------------- +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:55:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489884 41.613522 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.959961 68.039062 67.933594 +util 95.750000 99.000000 93.000000 +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 798 : avg iteration time 46.11 seconds +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 798 : data loading time 0.00 +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 798 : avg forward pass time 12.09 +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 798 : backward pass time 10.16 +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 798 : optimizer step time 1.59 +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 798-------------------- +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489888 41.613525 41.431000 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.945312 68.039062 67.914062 +util 93.250000 97.000000 90.000000 +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 799 : avg iteration time 45.97 seconds +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 799 : data loading time 0.00 +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 799 : avg forward pass time 12.05 +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 799 : backward pass time 10.15 +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 799 : optimizer step time 1.56 +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 799-------------------- +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:57:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489574 41.612228 41.431019 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.032715 136.037109 136.031250 +util 66.750000 71.000000 58.000000 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 800 : avg iteration time 162.91 seconds +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 800 : data loading time 0.00 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 800 : avg forward pass time 72.86 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 800 : backward pass time 8.43 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 800 : optimizer step time 0.40 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 800-------------------- +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 905671.8945 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4745 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 905671.8945iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7692iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.3403 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.8119 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2090616.2500iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 1.9752 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 45071996.0000iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4745iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.7869 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5802443.0000iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.2673 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2052389.7500iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.7246 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 693302.4375iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 8.0207 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 9261761.0000iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 10:59:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 11:00:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.69 seconds +[May 5, 2026 - 11:00:10 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 800 +[May 5, 2026 - 11:00:10 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000800 +[May 5, 2026 - 11:00:28 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 800 +[May 5, 2026 - 11:00:28 | INFO | fastgen.trainer:validate:358 ] Validation iteration 800 +[May 5, 2026 - 11:00:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:00:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.0s +[May 5, 2026 - 11:00:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=5.968 +[May 5, 2026 - 11:00:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:00:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:00:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.280 +[May 5, 2026 - 11:00:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:00:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:00:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.546 +[May 5, 2026 - 11:00:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:00:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:00:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.793 +[May 5, 2026 - 11:00:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:00:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:00:54 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.699 +[May 5, 2026 - 11:00:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:00:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:00:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.348 +[May 5, 2026 - 11:00:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:01:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:01:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.007 +[May 5, 2026 - 11:01:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:01:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:01:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.448 +[May 5, 2026 - 11:01:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:01:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:01:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.048 +[May 5, 2026 - 11:01:14 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 11:01:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 11:01:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.790 +[May 5, 2026 - 11:01:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 800-------------------- +[May 5, 2026 - 11:01:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 11:01:35 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 800 +[May 5, 2026 - 11:01:35 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.593 (n=10) +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491335 41.620167 41.430706 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.769043 68.564453 67.503906 +util 90.250000 94.000000 87.000000 +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 801 : avg iteration time 147.25 seconds +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 801 : data loading time 0.00 +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 801 : avg forward pass time 13.00 +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 801 : backward pass time 10.15 +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 801 : optimizer step time 1.57 +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 801-------------------- +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:02:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491101 41.619190 41.430737 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.783691 68.623047 67.503906 +util 90.750000 94.000000 88.000000 +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 802 : avg iteration time 46.21 seconds +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 802 : data loading time 0.00 +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 802 : avg forward pass time 12.13 +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 802 : backward pass time 10.21 +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 802 : optimizer step time 1.58 +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 802-------------------- +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:03:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491094 41.619190 41.430706 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.773926 68.583984 67.503906 +util 99.000000 100.000000 97.000000 +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 803 : avg iteration time 46.11 seconds +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 803 : data loading time 0.00 +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 803 : avg forward pass time 12.08 +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 803 : backward pass time 10.19 +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 803 : optimizer step time 1.57 +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 803-------------------- +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:03:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491097 41.619190 41.430706 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.773926 68.583984 67.503906 +util 96.750000 99.000000 94.000000 +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 804 : avg iteration time 43.57 seconds +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 804 : data loading time 0.00 +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 804 : avg forward pass time 10.83 +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 804 : backward pass time 10.17 +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 804 : optimizer step time 1.57 +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 804-------------------- +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:04:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490356 41.614883 41.431156 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.062012 136.154297 136.031250 +util 60.500000 64.000000 57.000000 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 805 : avg iteration time 162.94 seconds +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 805 : data loading time 0.00 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 805 : avg forward pass time 72.87 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 805 : backward pass time 8.42 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 805 : optimizer step time 0.41 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 805-------------------- +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 272095.6008 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4397 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 272095.6008iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4847iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2567 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1044 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 515637.4375iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.3470iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10950134.0000iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4397iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7312 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 95076.0938iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.3313 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5782.9448iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1086 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 202252.7969iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.5066 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 150.3997iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:07:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490423 41.615128 41.431179 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.827637 67.919922 67.796875 +util 96.250000 100.000000 88.000000 +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 806 : avg iteration time 44.91 seconds +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 806 : data loading time 0.00 +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 806 : avg forward pass time 11.52 +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 806 : backward pass time 10.15 +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 806 : optimizer step time 1.57 +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 806-------------------- +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:08:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490677 41.616131 41.431175 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.013184 68.134766 67.972656 +util 94.750000 98.000000 92.000000 +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 807 : avg iteration time 45.92 seconds +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 807 : data loading time 0.00 +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 807 : avg forward pass time 12.04 +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 807 : backward pass time 10.13 +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 807 : optimizer step time 1.57 +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 807-------------------- +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:08:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490670 41.616131 41.431156 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.812988 67.919922 67.777344 +util 96.250000 99.000000 88.000000 +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 808 : avg iteration time 43.48 seconds +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 808 : data loading time 0.00 +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 808 : avg forward pass time 10.82 +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 808 : backward pass time 10.14 +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 808 : optimizer step time 1.56 +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 808-------------------- +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:09:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490679 41.616131 41.431168 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.891113 68.115234 67.816406 +util 96.500000 100.000000 88.000000 +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 809 : avg iteration time 46.00 seconds +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 809 : data loading time 0.00 +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 809 : avg forward pass time 12.08 +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 809 : backward pass time 10.12 +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 809 : optimizer step time 1.58 +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 809-------------------- +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:10:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490262 41.614464 41.431179 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.008301 136.011719 135.998047 +util 66.250000 70.000000 62.000000 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 810 : avg iteration time 168.36 seconds +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 810 : data loading time 0.00 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 810 : avg forward pass time 75.58 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 810 : backward pass time 8.41 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 810 : optimizer step time 0.40 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 810-------------------- +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 720622.4219 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5497 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 720622.4219iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.2838iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.8681 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5718 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1334750.6250iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 309.8587iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 27883326.0000iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5497iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6691 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 83976.4062iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.8821 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 950032.6875iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.2203 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1868549.3750iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2544 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 270712.8125iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:13:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490256 41.614464 41.431156 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.281250 67.902344 67.074219 +util 94.250000 99.000000 88.000000 +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 811 : avg iteration time 47.68 seconds +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 811 : data loading time 0.00 +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 811 : avg forward pass time 12.90 +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 811 : backward pass time 10.16 +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 811 : optimizer step time 1.58 +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 811-------------------- +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:13:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490694 41.616215 41.431156 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.276367 67.414062 67.230469 +util 96.500000 100.000000 88.000000 +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 812 : avg iteration time 48.76 seconds +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 812 : data loading time 0.00 +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 812 : avg forward pass time 13.42 +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 812 : backward pass time 10.16 +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 812 : optimizer step time 1.59 +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 812-------------------- +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:14:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490889 41.616959 41.431179 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.964844 68.089844 67.589844 +util 96.500000 100.000000 89.000000 +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 813 : avg iteration time 45.99 seconds +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 813 : data loading time 0.00 +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 813 : avg forward pass time 12.06 +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 813 : backward pass time 10.15 +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 813 : optimizer step time 1.57 +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 813-------------------- +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:15:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490953 41.617218 41.431179 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.096680 68.117188 68.089844 +util 96.250000 100.000000 88.000000 +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 814 : avg iteration time 43.41 seconds +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 814 : data loading time 0.00 +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 814 : avg forward pass time 10.76 +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 814 : backward pass time 10.15 +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 814 : optimizer step time 1.58 +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 814-------------------- +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:16:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490316 41.614685 41.431164 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.047363 136.154297 136.011719 +util 67.000000 72.000000 57.000000 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 815 : avg iteration time 163.02 seconds +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 815 : data loading time 0.00 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 815 : avg forward pass time 72.91 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 815 : backward pass time 8.41 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 815 : optimizer step time 0.40 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 815-------------------- +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 396314.0024 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4659 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 396314.0024iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9088iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6501 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5484 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1275660.0000iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.6703 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 26611734.0000iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4659iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.6501 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3.6703 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.8507 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 16340.5117iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.9303 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 19160.5566iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2336 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 35142.1445iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:18:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490325 41.614685 41.431198 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.140137 68.167969 68.056641 +util 97.500000 99.000000 95.000000 +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 816 : avg iteration time 47.94 seconds +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 816 : data loading time 0.00 +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 816 : avg forward pass time 13.03 +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 816 : backward pass time 10.15 +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 816 : optimizer step time 1.58 +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 816-------------------- +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:19:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490588 41.615700 41.431225 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.008789 68.031250 67.941406 +util 97.500000 100.000000 91.000000 +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 817 : avg iteration time 43.47 seconds +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 817 : data loading time 0.00 +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 817 : avg forward pass time 10.79 +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 817 : backward pass time 10.15 +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 817 : optimizer step time 1.57 +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 817-------------------- +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:20:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490904 41.616955 41.431202 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.793945 68.078125 67.699219 +util 97.000000 100.000000 89.000000 +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 818 : avg iteration time 48.59 seconds +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 818 : data loading time 0.00 +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 818 : avg forward pass time 13.36 +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 818 : backward pass time 10.16 +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 818 : optimizer step time 1.58 +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 818-------------------- +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:21:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490906 41.616955 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.150391 68.167969 68.097656 +util 97.250000 99.000000 95.000000 +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 819 : avg iteration time 48.55 seconds +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 819 : data loading time 0.00 +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 819 : avg forward pass time 13.34 +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 819 : backward pass time 10.15 +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 819 : optimizer step time 1.58 +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 819-------------------- +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:22:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490396 41.614891 41.431252 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.065430 136.173828 136.029297 +util 65.500000 74.000000 57.000000 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 820 : avg iteration time 165.69 seconds +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 820 : data loading time 0.00 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 820 : avg forward pass time 74.24 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 820 : backward pass time 8.42 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 820 : optimizer step time 0.40 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 820-------------------- +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 545950.0449 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4483 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 545950.0449iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1158iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8197 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3928 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1592931.3750iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.1523 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 19494308.0000iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4483iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.9923 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7.2754 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.1544 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 29996.0977iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.0577 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 24721.6973iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.1922 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 239051.3750iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:24:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490386 41.614872 41.431232 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.022949 68.056641 68.011719 +util 95.750000 99.000000 93.000000 +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 821 : avg iteration time 44.89 seconds +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 821 : data loading time 0.00 +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 821 : avg forward pass time 11.52 +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 821 : backward pass time 10.16 +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 821 : optimizer step time 1.56 +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 821-------------------- +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:25:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490596 41.615681 41.431252 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.730469 67.835938 67.414062 +util 92.500000 100.000000 88.000000 +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 822 : avg iteration time 46.32 seconds +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 822 : data loading time 0.00 +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 822 : avg forward pass time 12.22 +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 822 : backward pass time 10.15 +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 822 : optimizer step time 1.57 +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 822-------------------- +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:26:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490647 41.615921 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.847656 67.992188 67.414062 +util 96.750000 100.000000 89.000000 +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 823 : avg iteration time 45.96 seconds +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 823 : data loading time 0.00 +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 823 : avg forward pass time 12.05 +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 823 : backward pass time 10.16 +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 823 : optimizer step time 1.56 +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 823-------------------- +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:27:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490648 41.615925 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842773 67.992188 67.394531 +util 91.000000 95.000000 88.000000 +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 824 : avg iteration time 46.06 seconds +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 824 : data loading time 0.00 +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 824 : avg forward pass time 12.09 +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 824 : backward pass time 10.16 +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 824 : optimizer step time 1.58 +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 824-------------------- +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:27:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490051 41.613503 41.431252 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.115234 136.011719 +util 67.250000 71.000000 64.000000 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 825 : avg iteration time 162.69 seconds +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 825 : data loading time 0.00 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 825 : avg forward pass time 72.74 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 825 : backward pass time 8.41 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 825 : optimizer step time 0.40 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 825-------------------- +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 150412.9795 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4615 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 150412.9795iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5508iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7490 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7527 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 387369.6562iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.4726 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5418425.5000iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4615iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.5864 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1303.4894iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0138 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3064.5752iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.9128 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 136699.6562iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9235 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 18901.3145iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:30:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490081 41.613636 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.705566 68.076172 67.582031 +util 94.250000 98.000000 91.000000 +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 826 : avg iteration time 50.14 seconds +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 826 : data loading time 0.00 +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 826 : avg forward pass time 14.15 +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 826 : backward pass time 10.14 +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 826 : optimizer step time 1.56 +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 826-------------------- +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:31:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490400 41.614876 41.431252 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.876953 67.882812 67.875000 +util 92.750000 96.000000 90.000000 +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 827 : avg iteration time 45.99 seconds +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 827 : data loading time 0.00 +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 827 : avg forward pass time 12.04 +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 827 : backward pass time 10.15 +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 827 : optimizer step time 1.57 +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 827-------------------- +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:32:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490643 41.615879 41.431232 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.808594 68.078125 67.718750 +util 96.500000 99.000000 94.000000 +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 828 : avg iteration time 46.05 seconds +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 828 : data loading time 0.00 +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 828 : avg forward pass time 12.11 +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 828 : backward pass time 10.12 +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 828 : optimizer step time 1.57 +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 828-------------------- +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:33:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490643 41.615879 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.779297 67.902344 67.738281 +util 95.750000 100.000000 87.000000 +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 829 : avg iteration time 48.65 seconds +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 829 : data loading time 0.00 +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 829 : avg forward pass time 13.39 +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 829 : backward pass time 10.15 +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 829 : optimizer step time 1.57 +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 829-------------------- +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:33:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:36:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489943 41.613045 41.431240 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.062012 136.154297 136.031250 +util 64.000000 68.000000 60.000000 +[May 5, 2026 - 11:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 830 : avg iteration time 163.34 seconds +[May 5, 2026 - 11:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 830 : data loading time 0.00 +[May 5, 2026 - 11:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 830 : avg forward pass time 73.04 +[May 5, 2026 - 11:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 830 : backward pass time 8.42 +[May 5, 2026 - 11:36:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 830 : optimizer step time 0.41 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 830-------------------- +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 73895.6377 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5470 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 73895.6377iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5399iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9269 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1210 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 129457.1172iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 47.1693iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1531764.6250iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5470iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.7198 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 12576.0615iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.7879 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 263.9531iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.2659 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 686.5815iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.2668 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 687.9330iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:36:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489930 41.613033 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.783691 68.095703 67.679688 +util 94.000000 98.000000 91.000000 +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 831 : avg iteration time 50.17 seconds +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 831 : data loading time 0.00 +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 831 : avg forward pass time 14.14 +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 831 : backward pass time 10.16 +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 831 : optimizer step time 1.59 +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 831-------------------- +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:37:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490055 41.613533 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.730469 67.941406 67.660156 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 832 : avg iteration time 43.31 seconds +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 832 : data loading time 0.00 +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 832 : avg forward pass time 10.73 +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 832 : backward pass time 10.15 +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 832 : optimizer step time 1.57 +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 832-------------------- +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:38:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490055 41.613533 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.769531 68.097656 67.660156 +util 89.750000 93.000000 87.000000 +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 833 : avg iteration time 46.04 seconds +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 833 : data loading time 0.00 +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 833 : avg forward pass time 12.10 +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 833 : backward pass time 10.14 +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 833 : optimizer step time 1.58 +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 833-------------------- +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:38:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490375 41.614811 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.779297 68.136719 67.660156 +util 91.750000 95.000000 89.000000 +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 834 : avg iteration time 43.42 seconds +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 834 : data loading time 0.00 +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 834 : avg forward pass time 10.77 +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 834 : backward pass time 10.15 +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 834 : optimizer step time 1.58 +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 834-------------------- +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:39:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490129 41.613831 41.431229 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.042480 136.251953 135.972656 +util 65.250000 70.000000 56.000000 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 835 : avg iteration time 168.31 seconds +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 835 : data loading time 0.00 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 835 : avg forward pass time 75.53 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 835 : backward pass time 8.47 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 835 : optimizer step time 0.40 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 835-------------------- +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 283036.6338 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5441 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 283036.6338iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4387iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6503 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5416 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 411350.8750iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.6717 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3552551.7500iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5441iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8109 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 15089.3447iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.8137 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2053.8630iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.6954 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1621.1515iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.0473 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3277.0408iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:42:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490130 41.613831 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500488 67.900391 67.367188 +util 95.000000 99.000000 92.000000 +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 836 : avg iteration time 44.94 seconds +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 836 : data loading time 0.00 +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 836 : avg forward pass time 11.53 +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 836 : backward pass time 10.15 +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 836 : optimizer step time 1.58 +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 836-------------------- +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:43:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490439 41.615070 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.383789 67.902344 67.210938 +util 91.000000 95.000000 88.000000 +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 837 : avg iteration time 45.94 seconds +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 837 : data loading time 0.00 +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 837 : avg forward pass time 12.03 +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 837 : backward pass time 10.16 +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 837 : optimizer step time 1.58 +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 837-------------------- +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:43:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490434 41.615047 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.510742 67.882812 67.386719 +util 94.000000 98.000000 91.000000 +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 838 : avg iteration time 43.44 seconds +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 838 : data loading time 0.00 +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 838 : avg forward pass time 10.79 +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 838 : backward pass time 10.14 +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 838 : optimizer step time 1.58 +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 838-------------------- +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:44:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490496 41.615295 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.383789 67.902344 67.210938 +util 97.500000 100.000000 95.000000 +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 839 : avg iteration time 45.85 seconds +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 839 : data loading time 0.00 +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 839 : avg forward pass time 12.00 +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 839 : backward pass time 10.13 +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 839 : optimizer step time 1.58 +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 839-------------------- +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:45:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490336 41.614658 41.431229 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.075195 136.154297 136.048828 +util 64.500000 68.000000 61.000000 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 840 : avg iteration time 162.98 seconds +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 840 : data loading time 0.00 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 840 : avg forward pass time 72.86 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 840 : backward pass time 8.47 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 840 : optimizer step time 0.40 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 840-------------------- +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 109241.7971 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5251 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 109241.7971iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0737iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.8933 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1290 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 230501.3438iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 325.8932iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1556637.3750iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5251iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.8885 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 130220.3438iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.3944 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 358158.7500iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.8933 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 325.8932iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6299 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1421.9530iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:48:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490339 41.614670 41.431229 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.632324 68.134766 67.464844 +util 89.250000 93.000000 86.000000 +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 841 : avg iteration time 47.51 seconds +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 841 : data loading time 0.00 +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 841 : avg forward pass time 12.81 +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 841 : backward pass time 10.16 +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 841 : optimizer step time 1.58 +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 841-------------------- +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:48:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490342 41.614647 41.431255 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.413086 68.078125 67.191406 +util 89.250000 93.000000 86.000000 +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 842 : avg iteration time 46.00 seconds +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 842 : data loading time 0.00 +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 842 : avg forward pass time 12.07 +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 842 : backward pass time 10.15 +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 842 : optimizer step time 1.57 +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 842-------------------- +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:49:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490402 41.614899 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.286133 67.511719 67.210938 +util 89.250000 93.000000 86.000000 +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 843 : avg iteration time 46.01 seconds +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 843 : data loading time 0.00 +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 843 : avg forward pass time 12.08 +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 843 : backward pass time 10.14 +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 843 : optimizer step time 1.57 +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 843-------------------- +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:50:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490423 41.614971 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.447266 68.097656 67.230469 +util 97.250000 100.000000 91.000000 +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 844 : avg iteration time 46.17 seconds +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 844 : data loading time 0.00 +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 844 : avg forward pass time 12.14 +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 844 : backward pass time 10.14 +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 844 : optimizer step time 1.59 +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 844-------------------- +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:51:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490205 41.614098 41.431240 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.099609 136.134766 136.087891 +util 59.250000 63.000000 56.000000 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 845 : avg iteration time 160.45 seconds +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 845 : data loading time 0.00 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 845 : avg forward pass time 71.64 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 845 : backward pass time 8.41 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 845 : optimizer step time 0.40 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 845-------------------- +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 348267.9512 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4476 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 348267.9512iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4560iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6345 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0955 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 816718.2500iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.5573 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10756092.0000iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4476iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.3136 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 13.8337 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.0212 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 22978.5059iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.0685 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 3418.6567iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.1163 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 9.3246 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:53:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490200 41.614075 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.446777 68.095703 67.230469 +util 95.000000 99.000000 92.000000 +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 846 : avg iteration time 50.25 seconds +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 846 : data loading time 0.00 +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 846 : avg forward pass time 14.19 +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 846 : backward pass time 10.15 +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 846 : optimizer step time 1.58 +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 846-------------------- +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0169 iter count: 1.0 +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0169 iter count: 1.0 +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:54:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490388 41.614826 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.618164 67.902344 67.523438 +util 90.250000 94.000000 87.000000 +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 847 : avg iteration time 43.35 seconds +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 847 : data loading time 0.00 +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 847 : avg forward pass time 10.74 +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 847 : backward pass time 10.15 +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 847 : optimizer step time 1.59 +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 847-------------------- +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0154 iter count: 1.0 +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:55:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490453 41.615078 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.413086 67.902344 67.250000 +util 89.750000 93.000000 87.000000 +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 848 : avg iteration time 43.39 seconds +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 848 : data loading time 0.00 +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 848 : avg forward pass time 10.78 +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 848 : backward pass time 10.14 +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 848 : optimizer step time 1.56 +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 848-------------------- +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:56:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490465 41.615097 41.431252 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.603516 67.902344 67.503906 +util 92.000000 96.000000 89.000000 +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 849 : avg iteration time 46.03 seconds +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 849 : data loading time 0.00 +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 849 : avg forward pass time 12.08 +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 849 : backward pass time 10.14 +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 849 : optimizer step time 1.57 +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 849-------------------- +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:56:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490190 41.614006 41.431240 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.062012 136.154297 136.031250 +util 61.250000 65.000000 58.000000 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 850 : avg iteration time 163.02 seconds +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 850 : data loading time 0.00 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 850 : avg forward pass time 72.91 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 850 : backward pass time 8.39 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 850 : optimizer step time 0.40 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 850-------------------- +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 227579.3708 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4449 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 227579.3708iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1052iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7118 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3227 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 565533.1250iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.1522 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 16942312.0000iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4449iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.0970 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 489.8249iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6202 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 76143.5391iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.8692 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 42.0310 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1310 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 28624.9336iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 11:59:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490185 41.614010 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.930176 67.933594 67.919922 +util 97.250000 100.000000 90.000000 +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 851 : avg iteration time 44.93 seconds +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 851 : data loading time 0.00 +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 851 : avg forward pass time 11.54 +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 851 : backward pass time 10.14 +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 851 : optimizer step time 1.57 +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 851-------------------- +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:00:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490314 41.614525 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.896484 68.078125 67.835938 +util 98.250000 100.000000 96.000000 +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 852 : avg iteration time 45.95 seconds +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 852 : data loading time 0.00 +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 852 : avg forward pass time 12.03 +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 852 : backward pass time 10.16 +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 852 : optimizer step time 1.57 +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 852-------------------- +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:01:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490328 41.614525 41.431252 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.896484 68.078125 67.835938 +util 94.750000 100.000000 88.000000 +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 853 : avg iteration time 43.47 seconds +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 853 : data loading time 0.00 +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 853 : avg forward pass time 10.80 +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 853 : backward pass time 10.15 +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 853 : optimizer step time 1.57 +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 853-------------------- +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:01:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490330 41.614540 41.431255 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.896484 68.078125 67.835938 +util 90.750000 95.000000 88.000000 +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 854 : avg iteration time 46.03 seconds +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 854 : data loading time 0.00 +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 854 : avg forward pass time 12.06 +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 854 : backward pass time 10.17 +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 854 : optimizer step time 1.57 +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 854-------------------- +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:02:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489982 41.613152 41.431248 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.060547 136.095703 136.048828 +util 59.250000 63.000000 54.000000 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 855 : avg iteration time 168.40 seconds +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 855 : data loading time 0.00 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 855 : avg forward pass time 75.60 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 855 : backward pass time 8.38 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 855 : optimizer step time 0.41 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 855-------------------- +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 127472.4503 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4362 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 127472.4503iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6052iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3262 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7615 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 290513.6875iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 14.1869iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5514771.0000iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4362iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7436 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 97454.4844iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.6901 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1603.8606iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4769 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1047.1893iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2951 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 39744.4023iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:05:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490041 41.613396 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.944824 68.095703 67.894531 +util 97.000000 99.000000 91.000000 +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 856 : avg iteration time 47.55 seconds +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 856 : data loading time 0.00 +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 856 : avg forward pass time 12.83 +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 856 : backward pass time 10.14 +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 856 : optimizer step time 1.58 +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 856-------------------- +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0145 iter count: 1.0 +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:06:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490495 41.615166 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.003906 68.031250 67.921875 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 857 : avg iteration time 43.40 seconds +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 857 : data loading time 0.00 +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 857 : avg forward pass time 10.75 +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 857 : backward pass time 10.15 +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 857 : optimizer step time 1.57 +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 857-------------------- +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:07:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490552 41.615398 41.431259 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.798828 67.921875 67.757812 +util 95.250000 99.000000 92.000000 +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 858 : avg iteration time 45.91 seconds +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 858 : data loading time 0.00 +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 858 : avg forward pass time 12.02 +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 858 : backward pass time 10.16 +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 858 : optimizer step time 1.58 +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 858-------------------- +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:07:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490453 41.614998 41.431259 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.798828 67.921875 67.757812 +util 95.000000 100.000000 89.000000 +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 859 : avg iteration time 45.93 seconds +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 859 : data loading time 0.00 +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 859 : avg forward pass time 12.03 +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 859 : backward pass time 10.14 +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 859 : optimizer step time 1.57 +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 859-------------------- +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:08:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490214 41.614071 41.431240 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.993652 136.056641 135.972656 +util 61.750000 73.000000 54.000000 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 860 : avg iteration time 162.81 seconds +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 860 : data loading time 0.00 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 860 : avg forward pass time 72.79 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 860 : backward pass time 8.42 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 860 : optimizer step time 0.40 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 860-------------------- +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 259181.8359 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4690 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 259181.8359iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7534iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5198 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7386 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 480736.7188iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.8279 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5267799.5000iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4690iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.7004 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4880344.0000iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1974 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4424.2642iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.7386 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 5267799.5000iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.7812 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 776411.5625iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:11:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490220 41.614090 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.476074 67.919922 67.328125 +util 96.500000 99.000000 89.000000 +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 861 : avg iteration time 50.14 seconds +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 861 : data loading time 0.00 +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 861 : avg forward pass time 14.13 +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 861 : backward pass time 10.17 +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 861 : optimizer step time 1.57 +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 861-------------------- +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0148 iter count: 1.0 +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:12:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490467 41.615078 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.696289 68.097656 67.562500 +util 97.500000 100.000000 95.000000 +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 862 : avg iteration time 43.36 seconds +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 862 : data loading time 0.00 +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 862 : avg forward pass time 10.75 +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 862 : backward pass time 10.13 +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 862 : optimizer step time 1.57 +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 862-------------------- +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0138 iter count: 1.0 +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:12:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490467 41.615078 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.784180 67.921875 67.738281 +util 96.000000 99.000000 88.000000 +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 863 : avg iteration time 46.03 seconds +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 863 : data loading time 0.00 +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 863 : avg forward pass time 12.09 +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 863 : backward pass time 10.15 +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 863 : optimizer step time 1.56 +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 863-------------------- +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:13:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490468 41.615078 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.593750 67.628906 67.582031 +util 97.250000 100.000000 95.000000 +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 864 : avg iteration time 45.97 seconds +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 864 : data loading time 0.00 +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 864 : avg forward pass time 12.04 +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 864 : backward pass time 10.16 +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 864 : optimizer step time 1.59 +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 864-------------------- +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:14:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490092 41.613575 41.431240 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.042480 136.076172 136.031250 +util 68.500000 73.000000 65.000000 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 865 : avg iteration time 163.03 seconds +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 865 : data loading time 0.00 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 865 : avg forward pass time 72.89 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 865 : backward pass time 8.46 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 865 : optimizer step time 0.40 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 865-------------------- +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 520275.1289 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4289 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 520275.1289iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1229iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1035 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2131 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1146514.5000iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.0881 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 13607248.0000iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4289iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.9945 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 21785.9668iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.8387 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 117887.9922iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2938 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 39643.8945iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2033 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 33078.8594iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:17:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490093 41.613575 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.223145 68.265625 68.095703 +util 95.500000 99.000000 93.000000 +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 866 : avg iteration time 47.55 seconds +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 866 : data loading time 0.00 +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 866 : avg forward pass time 12.87 +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 866 : backward pass time 10.12 +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 866 : optimizer step time 1.56 +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 866-------------------- +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:17:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490340 41.614578 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.706055 67.902344 67.640625 +util 94.250000 99.000000 88.000000 +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 867 : avg iteration time 48.50 seconds +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 867 : data loading time 0.00 +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 867 : avg forward pass time 13.33 +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 867 : backward pass time 10.14 +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 867 : optimizer step time 1.56 +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 867-------------------- +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:18:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490336 41.614578 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.174805 68.265625 67.902344 +util 93.750000 98.000000 91.000000 +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 868 : avg iteration time 48.68 seconds +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 868 : data loading time 0.00 +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 868 : avg forward pass time 13.43 +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 868 : backward pass time 10.13 +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 868 : optimizer step time 1.57 +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 868-------------------- +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:19:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490342 41.614578 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.706055 67.902344 67.640625 +util 94.750000 99.000000 92.000000 +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 869 : avg iteration time 46.01 seconds +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 869 : data loading time 0.00 +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 869 : avg forward pass time 12.07 +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 869 : backward pass time 10.13 +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 869 : optimizer step time 1.58 +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 869-------------------- +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:20:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489958 41.613033 41.431240 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.075195 136.154297 136.048828 +util 60.000000 64.000000 57.000000 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 870 : avg iteration time 168.16 seconds +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 870 : data loading time 0.00 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 870 : avg forward pass time 75.43 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 870 : backward pass time 8.41 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 870 : optimizer step time 0.40 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 870-------------------- +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 258263.6328 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5401 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 258263.6328iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7222iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6950 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0737 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 552566.1250iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.0148 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10296672.0000iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5401iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1208 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 513.6739iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.8564 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 902433.7500iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.9108 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 337.4904iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.7470 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 98119.0234iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:23:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489958 41.613033 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.018066 68.154297 67.972656 +util 96.000000 100.000000 88.000000 +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 871 : avg iteration time 50.31 seconds +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 871 : data loading time 0.00 +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 871 : avg forward pass time 14.23 +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 871 : backward pass time 10.14 +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 871 : optimizer step time 1.57 +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 871-------------------- +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:23:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490147 41.613789 41.431263 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.086914 68.148438 67.902344 +util 98.250000 100.000000 96.000000 +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 872 : avg iteration time 43.39 seconds +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 872 : data loading time 0.00 +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 872 : avg forward pass time 10.75 +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 872 : backward pass time 10.16 +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 872 : optimizer step time 1.58 +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 872-------------------- +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:24:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490141 41.613789 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.891602 67.894531 67.882812 +util 94.750000 99.000000 92.000000 +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 873 : avg iteration time 45.95 seconds +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 873 : data loading time 0.00 +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 873 : avg forward pass time 12.05 +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 873 : backward pass time 10.13 +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 873 : optimizer step time 1.57 +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 873-------------------- +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:25:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490147 41.613789 41.431240 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.101562 68.167969 67.902344 +util 92.750000 97.000000 90.000000 +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 874 : avg iteration time 45.98 seconds +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 874 : data loading time 0.00 +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 874 : avg forward pass time 12.05 +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 874 : backward pass time 10.15 +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 874 : optimizer step time 1.58 +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 874-------------------- +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:26:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489951 41.612881 41.431316 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.096191 136.232422 136.050781 +util 65.250000 70.000000 57.000000 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 875 : avg iteration time 168.11 seconds +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 875 : data loading time 0.00 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 875 : avg forward pass time 75.47 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 875 : backward pass time 8.41 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 875 : optimizer step time 0.40 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 875-------------------- +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 2060148.9121 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5285 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 2060148.9121iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8215iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7595 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.2461 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 5175319.0000iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.5673 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 107420736.0000iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5285iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.6584 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 11124.2549iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.9899 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1178473.6250iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2019 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4464.2744iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9032 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 18150.2285iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:28:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.489947 41.612881 41.431324 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.627441 67.939453 67.523438 +util 90.750000 95.000000 88.000000 +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 876 : avg iteration time 47.67 seconds +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 876 : data loading time 0.00 +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 876 : avg forward pass time 12.89 +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 876 : backward pass time 10.14 +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 876 : optimizer step time 1.58 +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 876-------------------- +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:29:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490381 41.614632 41.431301 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.623047 67.921875 67.523438 +util 94.750000 99.000000 89.000000 +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 877 : avg iteration time 43.37 seconds +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 877 : data loading time 0.00 +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 877 : avg forward pass time 10.75 +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 877 : backward pass time 10.15 +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 877 : optimizer step time 1.57 +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 877-------------------- +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:30:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490447 41.614880 41.431301 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.437500 67.941406 67.269531 +util 96.750000 100.000000 89.000000 +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 878 : avg iteration time 43.36 seconds +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 878 : data loading time 0.00 +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 878 : avg forward pass time 10.75 +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 878 : backward pass time 10.16 +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 878 : optimizer step time 1.57 +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 878-------------------- +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:31:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490505 41.615128 41.431301 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.750000 67.960938 67.679688 +util 97.250000 100.000000 91.000000 +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 879 : avg iteration time 43.42 seconds +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 879 : data loading time 0.00 +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 879 : avg forward pass time 10.77 +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 879 : backward pass time 10.16 +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 879 : optimizer step time 1.56 +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 879-------------------- +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:31:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490214 41.613380 41.431469 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.115234 136.011719 +util 63.750000 68.000000 60.000000 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 880 : avg iteration time 168.29 seconds +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 880 : data loading time 0.00 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 880 : avg forward pass time 75.46 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 880 : backward pass time 8.47 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 880 : optimizer step time 0.40 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 880-------------------- +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 776203.6406 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4855 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 776203.6406iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1098iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.3045 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.4913 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1493319.7500iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 100.3779iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 23739496.0000iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4855iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.5906 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 9712.7090iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.3229 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 310438.0312iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6901 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 11850.2256iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4864 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 7885.3442iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:34:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490224 41.613388 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.793457 68.134766 67.679688 +util 97.500000 100.000000 95.000000 +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 881 : avg iteration time 45.03 seconds +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 881 : data loading time 0.00 +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 881 : avg forward pass time 11.59 +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 881 : backward pass time 10.13 +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 881 : optimizer step time 1.58 +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 881-------------------- +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0157 iter count: 1.0 +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0157 iter count: 1.0 +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:35:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490533 41.614639 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.325195 67.902344 67.132812 +util 96.000000 100.000000 88.000000 +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 882 : avg iteration time 48.62 seconds +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 882 : data loading time 0.00 +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 882 : avg forward pass time 13.39 +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 882 : backward pass time 10.14 +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 882 : optimizer step time 1.57 +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 882-------------------- +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:36:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490786 41.615639 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471680 67.902344 67.328125 +util 95.750000 99.000000 93.000000 +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 883 : avg iteration time 43.35 seconds +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 883 : data loading time 0.00 +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 883 : avg forward pass time 10.75 +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 883 : backward pass time 10.13 +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 883 : optimizer step time 1.57 +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 883-------------------- +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:36:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490862 41.615913 41.431484 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.222656 67.492188 67.132812 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 884 : avg iteration time 46.07 seconds +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 884 : data loading time 0.00 +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 884 : avg forward pass time 12.10 +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 884 : backward pass time 10.13 +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 884 : optimizer step time 1.58 +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 884-------------------- +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:37:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490273 41.613586 41.431469 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.081543 136.115234 136.070312 +util 65.000000 69.000000 61.000000 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 885 : avg iteration time 165.72 seconds +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 885 : data loading time 0.00 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 885 : avg forward pass time 74.20 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 885 : backward pass time 8.49 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 885 : optimizer step time 0.40 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 885-------------------- +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 119528.8516 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4452 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 119528.8516iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4905iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2751 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5292 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 243721.2812iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 94.6562iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3465448.2500iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4452iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.9314 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 351.7279iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.1031 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 495.7982iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8616 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 123396.7188iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.1050 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3677.6233iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:40:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490279 41.613586 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.832520 67.939453 67.796875 +util 95.250000 99.000000 92.000000 +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 886 : avg iteration time 47.59 seconds +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 886 : data loading time 0.00 +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 886 : avg forward pass time 12.85 +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 886 : backward pass time 10.16 +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 886 : optimizer step time 1.56 +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 886-------------------- +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0159 iter count: 1.0 +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:41:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490714 41.615345 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.969727 67.972656 67.960938 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 887 : avg iteration time 48.69 seconds +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 887 : data loading time 0.00 +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 887 : avg forward pass time 13.40 +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 887 : backward pass time 10.15 +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 887 : optimizer step time 1.58 +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 887-------------------- +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:42:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490848 41.615868 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.194336 68.226562 68.097656 +util 94.500000 99.000000 91.000000 +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 888 : avg iteration time 46.10 seconds +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 888 : data loading time 0.00 +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 888 : avg forward pass time 12.10 +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 888 : backward pass time 10.16 +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 888 : optimizer step time 1.59 +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 888-------------------- +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:42:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490904 41.616096 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.038086 68.117188 68.011719 +util 98.000000 100.000000 96.000000 +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 889 : avg iteration time 46.05 seconds +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 889 : data loading time 0.00 +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 889 : avg forward pass time 12.08 +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 889 : backward pass time 10.15 +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 889 : optimizer step time 1.59 +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 889-------------------- +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:43:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490390 41.614048 41.431469 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.065430 136.115234 136.048828 +util 60.250000 63.000000 55.000000 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 890 : avg iteration time 165.50 seconds +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 890 : data loading time 0.00 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 890 : avg forward pass time 74.17 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 890 : backward pass time 8.41 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 890 : optimizer step time 0.41 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 890-------------------- +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1762185.1323 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4948 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1762185.1323iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7523iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5142 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.1574 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 3520623.2500iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.7964 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 89961448.0000iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4948iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.4302 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 52073.2617iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.0344 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 23594.6484iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.5255 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1154.1251iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.7682 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 13854.7393iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:46:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490395 41.614029 41.431507 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.310059 67.900391 67.113281 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 891 : avg iteration time 47.53 seconds +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 891 : data loading time 0.00 +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 891 : avg forward pass time 12.83 +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 891 : backward pass time 10.14 +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 891 : optimizer step time 1.58 +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 891-------------------- +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:47:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491108 41.616920 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.160156 68.246094 67.902344 +util 94.750000 100.000000 88.000000 +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 892 : avg iteration time 46.02 seconds +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 892 : data loading time 0.00 +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 892 : avg forward pass time 12.09 +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 892 : backward pass time 10.14 +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 892 : optimizer step time 1.57 +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 892-------------------- +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:47:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491136 41.617031 41.431469 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.130859 68.148438 68.078125 +util 92.000000 96.000000 89.000000 +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 893 : avg iteration time 48.66 seconds +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 893 : data loading time 0.00 +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 893 : avg forward pass time 13.38 +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 893 : backward pass time 10.16 +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 893 : optimizer step time 1.58 +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 893-------------------- +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:48:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491151 41.617085 41.431477 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.847656 68.117188 67.757812 +util 94.250000 100.000000 87.000000 +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 894 : avg iteration time 45.93 seconds +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 894 : data loading time 0.00 +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 894 : avg forward pass time 12.05 +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 894 : backward pass time 10.12 +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 894 : optimizer step time 1.57 +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 894-------------------- +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:49:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490770 41.615562 41.431477 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.013184 136.076172 135.992188 +util 65.250000 70.000000 62.000000 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 895 : avg iteration time 165.52 seconds +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 895 : data loading time 0.00 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 895 : avg forward pass time 74.17 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 895 : backward pass time 8.42 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 895 : optimizer step time 0.41 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 895-------------------- +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1751911.5586 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5076 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1751911.5586iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.3096iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.1674 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.1424 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 3585349.2500iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 76.3022iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 87297464.0000iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5076iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6944 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1617.7014iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9624 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 150976.0000iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3832 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 47401.4922iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.3792 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 347449.3438iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:52:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.490834 41.615753 41.431511 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.671387 68.056641 67.542969 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 896 : avg iteration time 50.26 seconds +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 896 : data loading time 0.00 +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 896 : avg forward pass time 14.20 +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 896 : backward pass time 10.16 +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 896 : optimizer step time 1.57 +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 896-------------------- +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:53:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491109 41.616879 41.431477 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.642578 67.941406 67.542969 +util 97.000000 100.000000 90.000000 +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 897 : avg iteration time 45.99 seconds +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 897 : data loading time 0.00 +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 897 : avg forward pass time 12.05 +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 897 : backward pass time 10.15 +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 897 : optimizer step time 1.56 +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 897-------------------- +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491116 41.616913 41.431477 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.642578 67.941406 67.542969 +util 91.750000 96.000000 89.000000 +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 898 : avg iteration time 48.68 seconds +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 898 : data loading time 0.00 +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 898 : avg forward pass time 13.41 +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 898 : backward pass time 10.14 +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 898 : optimizer step time 1.56 +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 898-------------------- +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:54:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.491125 41.616951 41.431477 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.642578 67.941406 67.542969 +util 97.250000 100.000000 90.000000 +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 899 : avg iteration time 48.73 seconds +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 899 : data loading time 0.00 +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 899 : avg forward pass time 13.42 +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 899 : backward pass time 10.17 +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 899 : optimizer step time 1.57 +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 899-------------------- +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 12:55:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 12:55:32 | INFO | fastgen.trainer:run:225 ] Training complete. +[May 5, 2026 - 12:55:32 | INFO | fastgen.trainer:validate:358 ] Validation iteration 900 +[May 5, 2026 - 12:55:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:55:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:55:37 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.076 +[May 5, 2026 - 12:55:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:55:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:55:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.301 +[May 5, 2026 - 12:55:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:55:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:55:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.342 +[May 5, 2026 - 12:55:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:55:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:55:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.268 +[May 5, 2026 - 12:55:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:55:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:55:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.667 +[May 5, 2026 - 12:55:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:56:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:56:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.139 +[May 5, 2026 - 12:56:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:56:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:56:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.205 +[May 5, 2026 - 12:56:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:56:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:56:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.293 +[May 5, 2026 - 12:56:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:56:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:56:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.190 +[May 5, 2026 - 12:56:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 12:56:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 12:56:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.545 +[May 5, 2026 - 12:56:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 900-------------------- +[May 5, 2026 - 12:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 12:56:40 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 900 +[May 5, 2026 - 12:56:40 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.603 (n=10) +[May 5, 2026 - 12:56:40 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 900 +[May 5, 2026 - 12:56:40 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900 +[May 5, 2026 - 12:56:58 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 900 +[May 5, 2026 - 12:56:59 | INFO | fastgen.trainer:run:232 ] Taking a 10 sec nap and exiting training. +[May 5, 2026 - 12:57:09 | INFO | fastgen.utils.distributed:clean_up:88 ] Distributed clean up. +[May 5, 2026 - 12:57:09 | INFO | __main__::49 ] Training finished. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2114024d758dfd344b8af269919677f9291822b9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-04T20:03:47.244974Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py", + "-", + "trainer.resume=True", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=900", + "trainer.save_ckpt_iter=100" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1176008974336" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "6zh9kmyzdkh7j025wtgawcwc8dwjekhq" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7692daa460e670008edbaad5788d733f7044ce72 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/files/wandb-summary.json @@ -0,0 +1 @@ +{"train_media/student/generation":{"sha256":"6a56be7cdd484957da3294100dc1510e28d6646918317e2af5b10224a8e8cae7","size":242492,"path":"media/videos/train_media/student/generation_800_6a56be7cdd484957da32.mp4","_type":"video-file"},"train/reward_sync_c_min":2.1673507690429688,"train/reward_weight_r1":150976,"val0/sync_c_sample_6":6.204550743103027,"train/gan_loss_gen":0,"val0/sync_c_sample_5":7.139270782470703,"optimizer/grad_norm (model_key net)":0,"train/reward_weight_r3":347449.34375,"train/reward_weight_max":8.7297464e+07,"train/reward_weight_mean":3.58534925e+06,"profiler/optimizer_step_time":1.5695875436067581,"_timestamp":1.7779534000288477e+09,"optimizer/iteration":900,"val0/total_loss":0,"train/reward_sync_c_max":9.142416000366211,"net/trainable_params":1421383840,"fake_score/trainable_params":1421383840,"fake_score/total_params":1421383840,"train/reward_weight_r2":47401.4921875,"profiler/avg_iteration_time":48.72523021697998,"profiler/data_loading_time":0.0009117927402257919,"val0/sync_c_sample_4":4.666830062866211,"val0/sync_c_sample_3":5.267613410949707,"model/local_total_params":[4299192568,4297394424,4297394424,4297394424],"optimizer/lr_net":2e-06,"val0/sync_c_sample_1":6.30142068862915,"model/total_params":17150480992,"train/reward_sync_c_r1":5.962438106536865,"optimizer/lr_fake_score":2e-06,"train/reward_sync_c_r3":6.379187107086182,"train/reward_weight_min":76.30217742919922,"val0/sync_c_sample_8":6.190480709075928,"profiler/avg_forward_pass_time":13.422852092422545,"val0/sync_c_sample_2":5.342367172241211,"val0/generated":{"captions":["sync_c=6.076","sync_c=6.301","sync_c=5.342","sync_c=5.268","sync_c=4.667","sync_c=7.139","sync_c=6.205","sync_c=5.293","sync_c=6.190","sync_c=3.545"],"_type":"videos","count":10,"videos":[{"caption":"sync_c=6.076","_type":"video-file","sha256":"564155b491ace85b197b991110d854637a4a392775a5d7e296926599f5a23aea","size":234191,"path":"media/videos/val0/generated_900_564155b491ace85b197b.mp4"},{"caption":"sync_c=6.301","_type":"video-file","sha256":"ba7693264ef940cbfab878518913a0813486cf7c93a8c73342a8f85566e9d3af","size":242361,"path":"media/videos/val0/generated_900_ba7693264ef940cbfab8.mp4"},{"sha256":"44d664992b102e09aee6485f2a7003031afc947fea0eaac81ca7902c93da4214","size":222605,"path":"media/videos/val0/generated_900_44d664992b102e09aee6.mp4","caption":"sync_c=5.342","_type":"video-file"},{"size":322596,"path":"media/videos/val0/generated_900_ec6cb67cea81ecc6beb8.mp4","caption":"sync_c=5.268","_type":"video-file","sha256":"ec6cb67cea81ecc6beb851ecfa4e610ffa389c4529db844000d306b923edd907"},{"size":282342,"path":"media/videos/val0/generated_900_5ff7b05030a5d107b25c.mp4","caption":"sync_c=4.667","_type":"video-file","sha256":"5ff7b05030a5d107b25c647d0cc8264f270b23da9b0a2b4ea43269182fd0d1b7"},{"_type":"video-file","sha256":"7c40be6e5027d8c71d060991c9fe8055ec4e4d75b2f03e1403f92f605fbec00e","size":200029,"path":"media/videos/val0/generated_900_7c40be6e5027d8c71d06.mp4","caption":"sync_c=7.139"},{"path":"media/videos/val0/generated_900_ac8f091c51ecec2b900d.mp4","caption":"sync_c=6.205","_type":"video-file","sha256":"ac8f091c51ecec2b900d57ed9390f7bd9055c0d6a2064882ba71829f2019f092","size":332355},{"_type":"video-file","sha256":"6050d77106d97831a9847c55efb8220f211302efbcc21bee2130906bbb137ef6","size":220475,"path":"media/videos/val0/generated_900_6050d77106d97831a984.mp4","caption":"sync_c=5.293"},{"size":256911,"path":"media/videos/val0/generated_900_b6a1e5b3f1258ffa6f86.mp4","caption":"sync_c=6.190","_type":"video-file","sha256":"b6a1e5b3f1258ffa6f8604e70a7e118b0d4aef6835a39d263b414fde7fd3c6d7"},{"size":265234,"path":"media/videos/val0/generated_900_0899e8e33a3daa243303.mp4","caption":"sync_c=3.545","_type":"video-file","sha256":"0899e8e33a3daa243303a494f14a0a91825f5b23602b2df0da23105351e60df3"}]},"val0/reconstructed":{"captions":false,"_type":"videos","count":10,"videos":[{"path":"media/videos/val0/reconstructed_900_80aab00fcfa85d5db98e.mp4","_type":"video-file","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028},{"_type":"video-file","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214","size":170922,"path":"media/videos/val0/reconstructed_900_884b2d197a5511393680.mp4"},{"_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8","size":173236,"path":"media/videos/val0/reconstructed_900_c0196720a45889bb2198.mp4"},{"path":"media/videos/val0/reconstructed_900_294040ac378ceaa34329.mp4","_type":"video-file","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372},{"sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004,"path":"media/videos/val0/reconstructed_900_15ac5b765a401d035db1.mp4","_type":"video-file"},{"_type":"video-file","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630,"path":"media/videos/val0/reconstructed_900_f5463a4ea6f68a39c5d0.mp4"},{"path":"media/videos/val0/reconstructed_900_c0f892b5fba78348af85.mp4","_type":"video-file","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986},{"sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516,"path":"media/videos/val0/reconstructed_900_a076d17e202ed18ebdf8.mp4","_type":"video-file"},{"_type":"video-file","sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001,"path":"media/videos/val0/reconstructed_900_6ff046418b5d41477989.mp4"},{"_type":"video-file","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490,"path":"media/videos/val0/reconstructed_900_f7412c9e170ffd299748.mp4"}]},"profiler/backward_pass_time":10.16517679579556,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"train/gan_loss_disc":0,"val0/sync_c_mean":5.602636051177979,"net/local_total_params":[355555624,355276072,355276072,355276072],"model/trainable_params":2842767680,"train/reward_sync_c_r0":3.694380760192871,"val0/sync_c_sample_7":5.292998313903809,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"train/reward_sync_c_mean":5.30959415435791,"train/fake_score_loss":0.011627197265625,"train_media/data/real":{"size":194172,"path":"media/videos/train_media/data/real_800_97779d11e00d4fea84a0.mp4","_type":"video-file","sha256":"97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2"},"net/local_trainable_params":[355555624,355276072,355276072,355276072],"net/total_params":1421383840,"train/reward_sync_c_r2":5.383204460144043,"model/local_trainable_params":[711111248,710552144,710552144,710552144],"val0/sync_c_sample_9":3.5453128814697266,"train/vsd_loss_unweighted":0.507568359375,"optimizer/non_finite_grads_count (model_key net)":0,"train/vsd_loss":0.507568359375,"val0/sync_c_sample_0":6.0755157470703125,"train/vsd_loss_weighted":1.75191155859375e+06,"_wandb":{"runtime":72103},"_runtime":72103.219844129,"train/total_loss":0.011627197265625,"_step":900,"train/reward_weight_r0":1617.701416015625} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..2912c9283b66948d6a369105ec940cbf630b4ad9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-05T05:03:47.293680363+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp4vtuo94y/port-3400577.txt","pid":3400577,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-05T05:03:47.294165657+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3400577} +{"time":"2026-05-05T05:03:47.294153121+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3400577-3411493-3597987039/socket","Net":"unix"}} +{"time":"2026-05-05T05:03:47.47971895+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-05T05:03:47.494892645+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T05:03:47.862718402+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T05:03:56.595517347+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ii11gea7kc5p"} +{"time":"2026-05-05T12:57:09.545815368+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T12:57:09.546004411+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T12:57:09.545998001+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T12:57:09.546054509+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T12:57:09.546074923+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3400577-3411493-3597987039/socket","Net":"unix"}} +{"time":"2026-05-05T12:57:11.55278209+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-05T12:57:11.552800547+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-05T12:57:11.552808254+09:00","level":"INFO","msg":"server is closed"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a0c91cbf758178804a9f021df95ddf7a45a74a7d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug-internal.log @@ -0,0 +1,644 @@ +{"time":"2026-05-05T05:03:47.49498206+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T05:03:47.86258599+09:00","level":"INFO","msg":"stream: created new stream","id":"lvwg9pfc"} +{"time":"2026-05-05T05:03:47.862687145+09:00","level":"INFO","msg":"handler: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T05:03:47.862700417+09:00","level":"INFO","msg":"stream: started","id":"lvwg9pfc"} +{"time":"2026-05-05T05:03:47.862713977+09:00","level":"INFO","msg":"sender: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T05:03:47.862721687+09:00","level":"INFO","msg":"writer: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T05:06:07.817652665+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":501,"current":604} +{"time":"2026-05-05T05:06:07.818121813+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":501,"current":604} +{"time":"2026-05-05T05:06:07.818419744+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":501,"current":604} +{"time":"2026-05-05T05:06:09.562332636+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":501,"current":604} +{"time":"2026-05-05T05:06:09.566346518+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":501,"current":604} +{"time":"2026-05-05T05:06:09.566695017+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":501,"current":604} +{"time":"2026-05-05T05:06:56.636653983+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":502,"current":604} +{"time":"2026-05-05T05:06:56.637173662+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":502,"current":604} +{"time":"2026-05-05T05:06:56.637423503+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":502,"current":604} +{"time":"2026-05-05T05:06:58.216608302+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":502,"current":604} +{"time":"2026-05-05T05:06:58.220463945+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":502,"current":604} +{"time":"2026-05-05T05:06:58.220829343+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":502,"current":604} +{"time":"2026-05-05T05:07:42.669796597+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":503,"current":604} +{"time":"2026-05-05T05:07:42.670303256+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":503,"current":604} +{"time":"2026-05-05T05:07:42.670556535+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":503,"current":604} +{"time":"2026-05-05T05:07:44.239402972+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":503,"current":604} +{"time":"2026-05-05T05:07:44.243264348+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":503,"current":604} +{"time":"2026-05-05T05:07:44.243604283+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":503,"current":604} +{"time":"2026-05-05T05:08:28.606603441+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":504,"current":604} +{"time":"2026-05-05T05:08:28.607155579+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":504,"current":604} +{"time":"2026-05-05T05:08:28.607410396+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":504,"current":604} +{"time":"2026-05-05T05:08:30.187846471+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":504,"current":604} +{"time":"2026-05-05T05:08:30.191651628+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":504,"current":604} +{"time":"2026-05-05T05:08:30.192014735+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":504,"current":604} +{"time":"2026-05-05T05:11:24.182307502+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":505,"current":604} +{"time":"2026-05-05T05:11:24.182815331+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":505,"current":604} +{"time":"2026-05-05T05:11:24.183075869+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":505,"current":604} +{"time":"2026-05-05T05:11:24.619116087+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":505,"current":604} +{"time":"2026-05-05T05:11:24.626452932+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":505,"current":604} +{"time":"2026-05-05T05:11:24.626920142+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":505,"current":604} +{"time":"2026-05-05T05:12:10.485430766+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":506,"current":604} +{"time":"2026-05-05T05:12:10.486102698+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":506,"current":604} +{"time":"2026-05-05T05:12:10.486422158+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":506,"current":604} +{"time":"2026-05-05T05:12:12.053268402+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":506,"current":604} +{"time":"2026-05-05T05:12:12.057114968+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":506,"current":604} +{"time":"2026-05-05T05:12:12.057468535+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":506,"current":604} +{"time":"2026-05-05T05:12:56.453188748+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":507,"current":604} +{"time":"2026-05-05T05:12:56.453724226+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":507,"current":604} +{"time":"2026-05-05T05:12:56.453993834+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":507,"current":604} +{"time":"2026-05-05T05:12:58.039602807+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":507,"current":604} +{"time":"2026-05-05T05:12:58.043483703+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":507,"current":604} +{"time":"2026-05-05T05:12:58.043851916+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":507,"current":604} +{"time":"2026-05-05T05:13:42.394702263+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":508,"current":604} +{"time":"2026-05-05T05:13:42.395121698+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":508,"current":604} +{"time":"2026-05-05T05:13:42.395430569+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":508,"current":604} +{"time":"2026-05-05T05:13:43.97003664+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":508,"current":604} +{"time":"2026-05-05T05:13:43.973928299+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":508,"current":604} +{"time":"2026-05-05T05:13:43.974260481+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":508,"current":604} +{"time":"2026-05-05T05:14:25.772789889+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":509,"current":604} +{"time":"2026-05-05T05:14:25.773304777+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":509,"current":604} +{"time":"2026-05-05T05:14:25.773573944+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":509,"current":604} +{"time":"2026-05-05T05:14:27.336624882+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":509,"current":604} +{"time":"2026-05-05T05:14:27.340445932+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":509,"current":604} +{"time":"2026-05-05T05:14:27.3408139+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":509,"current":604} +{"time":"2026-05-05T05:17:09.823587455+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":510,"current":604} +{"time":"2026-05-05T05:17:09.824111113+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":510,"current":604} +{"time":"2026-05-05T05:17:09.824368305+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":510,"current":604} +{"time":"2026-05-05T05:17:10.227352697+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":510,"current":604} +{"time":"2026-05-05T05:17:10.2348687+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":510,"current":604} +{"time":"2026-05-05T05:17:10.235334513+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":510,"current":604} +{"time":"2026-05-05T05:17:58.752816785+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":511,"current":604} +{"time":"2026-05-05T05:17:58.753411486+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":511,"current":604} +{"time":"2026-05-05T05:17:58.753661089+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":511,"current":604} +{"time":"2026-05-05T05:18:00.32236568+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":511,"current":604} +{"time":"2026-05-05T05:18:00.326202366+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":511,"current":604} +{"time":"2026-05-05T05:18:00.326559434+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":511,"current":604} +{"time":"2026-05-05T05:18:44.662779782+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":512,"current":604} +{"time":"2026-05-05T05:18:44.663297199+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":512,"current":604} +{"time":"2026-05-05T05:18:44.663532714+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":512,"current":604} +{"time":"2026-05-05T05:18:46.23591432+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":512,"current":604} +{"time":"2026-05-05T05:18:46.239687983+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":512,"current":604} +{"time":"2026-05-05T05:18:46.24004171+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":512,"current":604} +{"time":"2026-05-05T05:19:30.512151526+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":513,"current":604} +{"time":"2026-05-05T05:19:30.512674643+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":513,"current":604} +{"time":"2026-05-05T05:19:30.512913625+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":513,"current":604} +{"time":"2026-05-05T05:19:32.096745794+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":513,"current":604} +{"time":"2026-05-05T05:19:32.100567087+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":513,"current":604} +{"time":"2026-05-05T05:19:32.10094192+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":513,"current":604} +{"time":"2026-05-05T05:20:19.059869508+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":514,"current":604} +{"time":"2026-05-05T05:20:19.060384807+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":514,"current":604} +{"time":"2026-05-05T05:20:19.060624197+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":514,"current":604} +{"time":"2026-05-05T05:20:20.6319612+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":514,"current":604} +{"time":"2026-05-05T05:20:20.635991465+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":514,"current":604} +{"time":"2026-05-05T05:20:20.636346866+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":514,"current":604} +{"time":"2026-05-05T05:23:05.566398162+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":515,"current":604} +{"time":"2026-05-05T05:23:05.566924407+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":515,"current":604} +{"time":"2026-05-05T05:23:05.567169837+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":515,"current":604} +{"time":"2026-05-05T05:23:05.966111689+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":515,"current":604} +{"time":"2026-05-05T05:23:05.973568759+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":515,"current":604} +{"time":"2026-05-05T05:23:05.974040668+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":515,"current":604} +{"time":"2026-05-05T05:23:51.868331734+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":516,"current":604} +{"time":"2026-05-05T05:23:51.868782337+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":516,"current":604} +{"time":"2026-05-05T05:23:51.86910003+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":516,"current":604} +{"time":"2026-05-05T05:23:53.437051258+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":516,"current":604} +{"time":"2026-05-05T05:23:53.440881135+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":516,"current":604} +{"time":"2026-05-05T05:23:53.441238099+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":516,"current":604} +{"time":"2026-05-05T05:24:37.817135266+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":517,"current":604} +{"time":"2026-05-05T05:24:37.817640123+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":517,"current":604} +{"time":"2026-05-05T05:24:37.817915122+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":517,"current":604} +{"time":"2026-05-05T05:24:39.384822498+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":517,"current":604} +{"time":"2026-05-05T05:24:39.388607434+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":517,"current":604} +{"time":"2026-05-05T05:24:39.388964356+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":517,"current":604} +{"time":"2026-05-05T05:25:23.750610049+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":518,"current":604} +{"time":"2026-05-05T05:25:23.751124074+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":518,"current":604} +{"time":"2026-05-05T05:25:23.751401123+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":518,"current":604} +{"time":"2026-05-05T05:25:25.323443659+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":518,"current":604} +{"time":"2026-05-05T05:25:25.32736498+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":518,"current":604} +{"time":"2026-05-05T05:25:25.327702691+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":518,"current":604} +{"time":"2026-05-05T05:26:09.667616846+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":519,"current":604} +{"time":"2026-05-05T05:26:09.668108927+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":519,"current":604} +{"time":"2026-05-05T05:26:09.668379435+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":519,"current":604} +{"time":"2026-05-05T05:26:11.241244656+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":519,"current":604} +{"time":"2026-05-05T05:26:11.246598186+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":519,"current":604} +{"time":"2026-05-05T05:26:11.246968134+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":519,"current":604} +{"time":"2026-05-05T05:28:56.488634917+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":520,"current":604} +{"time":"2026-05-05T05:28:56.489152766+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":520,"current":604} +{"time":"2026-05-05T05:28:56.489402226+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":520,"current":604} +{"time":"2026-05-05T05:28:56.893691051+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":520,"current":604} +{"time":"2026-05-05T05:28:56.901115811+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":520,"current":604} +{"time":"2026-05-05T05:28:56.90158402+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":520,"current":604} +{"time":"2026-05-05T05:29:42.887132717+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":521,"current":604} +{"time":"2026-05-05T05:29:42.887601583+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":521,"current":604} +{"time":"2026-05-05T05:29:42.887897469+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":521,"current":604} +{"time":"2026-05-05T05:29:44.456336593+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":521,"current":604} +{"time":"2026-05-05T05:29:44.460293774+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":521,"current":604} +{"time":"2026-05-05T05:29:44.460558924+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":521,"current":604} +{"time":"2026-05-05T05:30:28.839636871+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":522,"current":604} +{"time":"2026-05-05T05:30:28.840176724+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":522,"current":604} +{"time":"2026-05-05T05:30:28.840428443+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":522,"current":604} +{"time":"2026-05-05T05:30:30.403597937+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":522,"current":604} +{"time":"2026-05-05T05:30:30.407450814+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":522,"current":604} +{"time":"2026-05-05T05:30:30.407776775+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":522,"current":604} +{"time":"2026-05-05T05:31:14.783016778+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":523,"current":604} +{"time":"2026-05-05T05:31:14.783542359+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":523,"current":604} +{"time":"2026-05-05T05:31:14.783788355+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":523,"current":604} +{"time":"2026-05-05T05:31:16.355183616+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":523,"current":604} +{"time":"2026-05-05T05:31:16.359030506+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":523,"current":604} +{"time":"2026-05-05T05:31:16.359402575+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":523,"current":604} +{"time":"2026-05-05T05:31:58.02260562+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":524,"current":604} +{"time":"2026-05-05T05:31:58.023084494+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":524,"current":604} +{"time":"2026-05-05T05:31:58.023332821+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":524,"current":604} +{"time":"2026-05-05T05:31:59.600483696+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":524,"current":604} +{"time":"2026-05-05T05:31:59.60432293+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":524,"current":604} +{"time":"2026-05-05T05:31:59.60466027+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":524,"current":604} +{"time":"2026-05-05T05:34:42.093611241+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":525,"current":604} +{"time":"2026-05-05T05:34:42.094026257+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":525,"current":604} +{"time":"2026-05-05T05:34:42.094345516+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":525,"current":604} +{"time":"2026-05-05T05:34:42.497175942+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":525,"current":604} +{"time":"2026-05-05T05:34:42.504637983+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":525,"current":604} +{"time":"2026-05-05T05:34:42.505106545+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":525,"current":604} +{"time":"2026-05-05T05:35:28.517234632+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":526,"current":604} +{"time":"2026-05-05T05:35:28.51783596+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":526,"current":604} +{"time":"2026-05-05T05:35:28.518098178+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":526,"current":604} +{"time":"2026-05-05T05:35:30.093690935+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":526,"current":604} +{"time":"2026-05-05T05:35:30.097595167+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":526,"current":604} +{"time":"2026-05-05T05:35:30.097981497+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":526,"current":604} +{"time":"2026-05-05T05:36:14.535947316+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":527,"current":604} +{"time":"2026-05-05T05:36:14.536443521+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":527,"current":604} +{"time":"2026-05-05T05:36:14.536691384+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":527,"current":604} +{"time":"2026-05-05T05:36:16.114749824+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":527,"current":604} +{"time":"2026-05-05T05:36:16.118533099+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":527,"current":604} +{"time":"2026-05-05T05:36:16.118889884+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":527,"current":604} +{"time":"2026-05-05T05:37:03.127534914+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":528,"current":604} +{"time":"2026-05-05T05:37:03.128019414+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":528,"current":604} +{"time":"2026-05-05T05:37:03.128321336+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":528,"current":604} +{"time":"2026-05-05T05:37:04.706007295+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":528,"current":604} +{"time":"2026-05-05T05:37:04.709839213+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":528,"current":604} +{"time":"2026-05-05T05:37:04.71016958+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":528,"current":604} +{"time":"2026-05-05T05:37:49.127396035+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":529,"current":604} +{"time":"2026-05-05T05:37:49.127976091+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":529,"current":604} +{"time":"2026-05-05T05:37:49.128248869+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":529,"current":604} +{"time":"2026-05-05T05:37:50.698737218+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":529,"current":604} +{"time":"2026-05-05T05:37:50.702753828+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":529,"current":604} +{"time":"2026-05-05T05:37:50.70306621+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":529,"current":604} +{"time":"2026-05-05T05:40:36.010616523+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":530,"current":604} +{"time":"2026-05-05T05:40:36.01119093+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":530,"current":604} +{"time":"2026-05-05T05:40:36.011435468+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":530,"current":604} +{"time":"2026-05-05T05:40:36.41259584+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":530,"current":604} +{"time":"2026-05-05T05:40:36.420557547+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":530,"current":604} +{"time":"2026-05-05T05:40:36.421055177+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":530,"current":604} +{"time":"2026-05-05T05:41:25.076875378+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":531,"current":604} +{"time":"2026-05-05T05:41:25.077392337+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":531,"current":604} +{"time":"2026-05-05T05:41:25.077661966+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":531,"current":604} +{"time":"2026-05-05T05:41:26.653895828+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":531,"current":604} +{"time":"2026-05-05T05:41:26.65775959+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":531,"current":604} +{"time":"2026-05-05T05:41:26.658079593+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":531,"current":604} +{"time":"2026-05-05T05:42:08.361130616+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":532,"current":604} +{"time":"2026-05-05T05:42:08.361596399+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":532,"current":604} +{"time":"2026-05-05T05:42:08.361877929+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":532,"current":604} +{"time":"2026-05-05T05:42:09.931925043+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":532,"current":604} +{"time":"2026-05-05T05:42:09.935779983+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":532,"current":604} +{"time":"2026-05-05T05:42:09.936085009+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":532,"current":604} +{"time":"2026-05-05T05:42:51.695144011+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":533,"current":604} +{"time":"2026-05-05T05:42:51.695652466+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":533,"current":604} +{"time":"2026-05-05T05:42:51.695929993+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":533,"current":604} +{"time":"2026-05-05T05:42:53.273709061+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":533,"current":604} +{"time":"2026-05-05T05:42:53.277543148+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":533,"current":604} +{"time":"2026-05-05T05:42:53.27788987+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":533,"current":604} +{"time":"2026-05-05T05:43:37.74386989+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":534,"current":604} +{"time":"2026-05-05T05:43:37.744396988+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":534,"current":604} +{"time":"2026-05-05T05:43:37.744672992+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":534,"current":604} +{"time":"2026-05-05T05:43:39.30079519+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":534,"current":604} +{"time":"2026-05-05T05:43:39.304680888+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":534,"current":604} +{"time":"2026-05-05T05:43:39.305039094+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":534,"current":604} +{"time":"2026-05-05T05:46:21.864198392+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":535,"current":604} +{"time":"2026-05-05T05:46:21.86480814+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":535,"current":604} +{"time":"2026-05-05T05:46:21.865081474+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":535,"current":604} +{"time":"2026-05-05T05:46:22.282874787+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":535,"current":604} +{"time":"2026-05-05T05:46:22.290386526+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":535,"current":604} +{"time":"2026-05-05T05:46:22.290856489+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":535,"current":604} +{"time":"2026-05-05T05:47:05.688604708+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":536,"current":604} +{"time":"2026-05-05T05:47:05.689133601+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":536,"current":604} +{"time":"2026-05-05T05:47:05.689403406+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":536,"current":604} +{"time":"2026-05-05T05:47:07.268749455+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":536,"current":604} +{"time":"2026-05-05T05:47:07.272606768+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":536,"current":604} +{"time":"2026-05-05T05:47:07.272971018+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":536,"current":604} +{"time":"2026-05-05T05:47:51.602239677+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":537,"current":604} +{"time":"2026-05-05T05:47:51.602753098+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":537,"current":604} +{"time":"2026-05-05T05:47:51.603029253+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":537,"current":604} +{"time":"2026-05-05T05:47:53.173486453+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":537,"current":604} +{"time":"2026-05-05T05:47:53.177300329+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":537,"current":604} +{"time":"2026-05-05T05:47:53.177653592+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":537,"current":604} +{"time":"2026-05-05T05:48:35.092259371+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":538,"current":604} +{"time":"2026-05-05T05:48:35.092782918+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":538,"current":604} +{"time":"2026-05-05T05:48:35.093023531+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":538,"current":604} +{"time":"2026-05-05T05:48:36.658502505+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":538,"current":604} +{"time":"2026-05-05T05:48:36.662394501+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":538,"current":604} +{"time":"2026-05-05T05:48:36.662768813+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":538,"current":604} +{"time":"2026-05-05T05:49:23.731550685+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":539,"current":604} +{"time":"2026-05-05T05:49:23.732059371+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":539,"current":604} +{"time":"2026-05-05T05:49:23.732330788+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":539,"current":604} +{"time":"2026-05-05T05:49:25.306018088+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":539,"current":604} +{"time":"2026-05-05T05:49:25.309888718+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":539,"current":604} +{"time":"2026-05-05T05:49:25.310244849+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":539,"current":604} +{"time":"2026-05-05T05:52:10.352583255+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":540,"current":604} +{"time":"2026-05-05T05:52:10.353108421+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":540,"current":604} +{"time":"2026-05-05T05:52:10.353371076+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":540,"current":604} +{"time":"2026-05-05T05:52:10.766885978+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":540,"current":604} +{"time":"2026-05-05T05:52:10.774383151+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":540,"current":604} +{"time":"2026-05-05T05:52:10.774872373+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":540,"current":604} +{"time":"2026-05-05T05:52:59.406515965+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":541,"current":604} +{"time":"2026-05-05T05:52:59.407030371+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":541,"current":604} +{"time":"2026-05-05T05:52:59.407320017+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":541,"current":604} +{"time":"2026-05-05T05:53:00.982871478+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":541,"current":604} +{"time":"2026-05-05T05:53:00.986833513+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":541,"current":604} +{"time":"2026-05-05T05:53:00.987182718+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":541,"current":604} +{"time":"2026-05-05T05:53:47.984146365+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":542,"current":604} +{"time":"2026-05-05T05:53:47.984709807+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":542,"current":604} +{"time":"2026-05-05T05:53:47.984970099+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":542,"current":604} +{"time":"2026-05-05T05:53:49.55270921+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":542,"current":604} +{"time":"2026-05-05T05:53:49.556302284+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":542,"current":604} +{"time":"2026-05-05T05:53:49.556641568+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":542,"current":604} +{"time":"2026-05-05T05:54:36.595658463+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":543,"current":604} +{"time":"2026-05-05T05:54:36.596171037+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":543,"current":604} +{"time":"2026-05-05T05:54:36.596458124+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":543,"current":604} +{"time":"2026-05-05T05:54:38.16189184+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":543,"current":604} +{"time":"2026-05-05T05:54:38.165685644+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":543,"current":604} +{"time":"2026-05-05T05:54:38.166003637+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":543,"current":604} +{"time":"2026-05-05T05:55:25.222060865+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":544,"current":604} +{"time":"2026-05-05T05:55:25.222611921+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":544,"current":604} +{"time":"2026-05-05T05:55:25.222876549+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":544,"current":604} +{"time":"2026-05-05T05:55:26.799828423+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":544,"current":604} +{"time":"2026-05-05T05:55:26.803694547+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":544,"current":604} +{"time":"2026-05-05T05:55:26.804049008+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":544,"current":604} +{"time":"2026-05-05T05:58:09.171494999+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":545,"current":604} +{"time":"2026-05-05T05:58:09.172004719+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":545,"current":604} +{"time":"2026-05-05T05:58:09.172254936+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":545,"current":604} +{"time":"2026-05-05T05:58:09.575764935+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":545,"current":604} +{"time":"2026-05-05T05:58:09.583140168+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":545,"current":604} +{"time":"2026-05-05T05:58:09.583611619+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":545,"current":604} +{"time":"2026-05-05T05:58:58.253291146+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":546,"current":604} +{"time":"2026-05-05T05:58:58.253830347+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":546,"current":604} +{"time":"2026-05-05T05:58:58.254086939+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":546,"current":604} +{"time":"2026-05-05T05:58:59.832181546+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":546,"current":604} +{"time":"2026-05-05T05:58:59.836134774+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":546,"current":604} +{"time":"2026-05-05T05:58:59.83649196+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":546,"current":604} +{"time":"2026-05-05T05:59:46.898359735+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":547,"current":604} +{"time":"2026-05-05T05:59:46.898879788+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":547,"current":604} +{"time":"2026-05-05T05:59:46.899115222+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":547,"current":604} +{"time":"2026-05-05T05:59:48.479016612+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":547,"current":604} +{"time":"2026-05-05T05:59:48.482873654+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":547,"current":604} +{"time":"2026-05-05T05:59:48.483186547+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":547,"current":604} +{"time":"2026-05-05T06:00:32.819041856+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":548,"current":604} +{"time":"2026-05-05T06:00:32.819528565+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":548,"current":604} +{"time":"2026-05-05T06:00:32.81982641+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":548,"current":604} +{"time":"2026-05-05T06:00:34.395793523+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":548,"current":604} +{"time":"2026-05-05T06:00:34.399665339+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":548,"current":604} +{"time":"2026-05-05T06:00:34.399977031+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":548,"current":604} +{"time":"2026-05-05T06:01:21.332155671+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":549,"current":604} +{"time":"2026-05-05T06:01:21.332648249+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":549,"current":604} +{"time":"2026-05-05T06:01:21.332933418+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":549,"current":604} +{"time":"2026-05-05T06:01:22.898758158+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":549,"current":604} +{"time":"2026-05-05T06:01:22.902644886+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":549,"current":604} +{"time":"2026-05-05T06:01:22.903531069+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":549,"current":604} +{"time":"2026-05-05T06:04:08.196710988+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":550,"current":604} +{"time":"2026-05-05T06:04:08.197216302+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":550,"current":604} +{"time":"2026-05-05T06:04:08.197493641+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":550,"current":604} +{"time":"2026-05-05T06:04:08.614874125+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":550,"current":604} +{"time":"2026-05-05T06:04:08.622303681+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":550,"current":604} +{"time":"2026-05-05T06:04:08.622779986+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":550,"current":604} +{"time":"2026-05-05T06:04:54.615349316+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":551,"current":604} +{"time":"2026-05-05T06:04:54.615849836+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":551,"current":604} +{"time":"2026-05-05T06:04:54.616123494+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":551,"current":604} +{"time":"2026-05-05T06:04:56.190120732+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":551,"current":604} +{"time":"2026-05-05T06:04:56.194016354+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":551,"current":604} +{"time":"2026-05-05T06:04:56.194308195+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":551,"current":604} +{"time":"2026-05-05T06:05:38.006811008+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":552,"current":604} +{"time":"2026-05-05T06:05:38.00735848+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":552,"current":604} +{"time":"2026-05-05T06:05:38.007630093+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":552,"current":604} +{"time":"2026-05-05T06:05:39.596980068+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":552,"current":604} +{"time":"2026-05-05T06:05:39.600872379+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":552,"current":604} +{"time":"2026-05-05T06:05:39.601245189+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":552,"current":604} +{"time":"2026-05-05T06:06:21.416410255+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":553,"current":604} +{"time":"2026-05-05T06:06:21.416947855+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":553,"current":604} +{"time":"2026-05-05T06:06:21.417239556+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":553,"current":604} +{"time":"2026-05-05T06:06:22.985111998+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":553,"current":604} +{"time":"2026-05-05T06:06:22.989029776+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":553,"current":604} +{"time":"2026-05-05T06:06:22.989416412+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":553,"current":604} +{"time":"2026-05-05T06:07:09.951367942+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":554,"current":604} +{"time":"2026-05-05T06:07:09.951886683+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":554,"current":604} +{"time":"2026-05-05T06:07:09.952131211+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":554,"current":604} +{"time":"2026-05-05T06:07:11.522487741+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":554,"current":604} +{"time":"2026-05-05T06:07:11.526302419+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":554,"current":604} +{"time":"2026-05-05T06:07:11.526655433+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":554,"current":604} +{"time":"2026-05-05T06:09:56.425527824+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":555,"current":604} +{"time":"2026-05-05T06:09:56.426044401+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":555,"current":604} +{"time":"2026-05-05T06:09:56.426304245+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":555,"current":604} +{"time":"2026-05-05T06:09:56.826848248+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":555,"current":604} +{"time":"2026-05-05T06:09:56.834284593+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":555,"current":604} +{"time":"2026-05-05T06:09:56.834744422+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":555,"current":604} +{"time":"2026-05-05T06:10:42.794712072+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":556,"current":604} +{"time":"2026-05-05T06:10:42.795228857+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":556,"current":604} +{"time":"2026-05-05T06:10:42.795513645+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":556,"current":604} +{"time":"2026-05-05T06:10:44.375949891+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":556,"current":604} +{"time":"2026-05-05T06:10:44.379815822+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":556,"current":604} +{"time":"2026-05-05T06:10:44.380289019+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":556,"current":604} +{"time":"2026-05-05T06:11:28.709997025+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":557,"current":604} +{"time":"2026-05-05T06:11:28.710518064+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":557,"current":604} +{"time":"2026-05-05T06:11:28.710788283+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":557,"current":604} +{"time":"2026-05-05T06:11:30.278969279+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":557,"current":604} +{"time":"2026-05-05T06:11:30.28278114+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":557,"current":604} +{"time":"2026-05-05T06:11:30.283118849+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":557,"current":604} +{"time":"2026-05-05T06:12:11.996907111+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":558,"current":604} +{"time":"2026-05-05T06:12:11.997413309+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":558,"current":604} +{"time":"2026-05-05T06:12:11.99766659+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":558,"current":604} +{"time":"2026-05-05T06:12:13.575497151+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":558,"current":604} +{"time":"2026-05-05T06:12:13.5794341+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":558,"current":604} +{"time":"2026-05-05T06:12:13.579794568+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":558,"current":604} +{"time":"2026-05-05T06:13:00.563504133+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":559,"current":604} +{"time":"2026-05-05T06:13:00.564066241+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":559,"current":604} +{"time":"2026-05-05T06:13:00.564317484+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":559,"current":604} +{"time":"2026-05-05T06:13:02.130127605+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":559,"current":604} +{"time":"2026-05-05T06:13:02.134076213+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":559,"current":604} +{"time":"2026-05-05T06:13:02.134444839+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":559,"current":604} +{"time":"2026-05-05T06:15:49.892065195+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":560,"current":604} +{"time":"2026-05-05T06:15:49.8926+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":560,"current":604} +{"time":"2026-05-05T06:15:49.892878027+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":560,"current":604} +{"time":"2026-05-05T06:15:50.292743303+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":560,"current":604} +{"time":"2026-05-05T06:15:50.300238956+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":560,"current":604} +{"time":"2026-05-05T06:15:50.300687929+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":560,"current":604} +{"time":"2026-05-05T06:16:36.273399033+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":561,"current":604} +{"time":"2026-05-05T06:16:36.273965195+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":561,"current":604} +{"time":"2026-05-05T06:16:36.274447626+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":561,"current":604} +{"time":"2026-05-05T06:16:37.844768564+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":561,"current":604} +{"time":"2026-05-05T06:16:37.848572317+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":561,"current":604} +{"time":"2026-05-05T06:16:37.848930975+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":561,"current":604} +{"time":"2026-05-05T06:17:19.598023946+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":562,"current":604} +{"time":"2026-05-05T06:17:19.598552175+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":562,"current":604} +{"time":"2026-05-05T06:17:19.598792814+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":562,"current":604} +{"time":"2026-05-05T06:17:21.175591413+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":562,"current":604} +{"time":"2026-05-05T06:17:21.179484705+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":562,"current":604} +{"time":"2026-05-05T06:17:21.179812105+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":562,"current":604} +{"time":"2026-05-05T06:18:08.197894065+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":563,"current":604} +{"time":"2026-05-05T06:18:08.198408861+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":563,"current":604} +{"time":"2026-05-05T06:18:08.198685812+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":563,"current":604} +{"time":"2026-05-05T06:18:09.766404996+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":563,"current":604} +{"time":"2026-05-05T06:18:09.77031496+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":563,"current":604} +{"time":"2026-05-05T06:18:09.770633518+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":563,"current":604} +{"time":"2026-05-05T06:18:51.491230496+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":564,"current":604} +{"time":"2026-05-05T06:18:51.491704474+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":564,"current":604} +{"time":"2026-05-05T06:18:51.491990122+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":564,"current":604} +{"time":"2026-05-05T06:18:53.07669924+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":564,"current":604} +{"time":"2026-05-05T06:18:53.080598535+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":564,"current":604} +{"time":"2026-05-05T06:18:53.080868403+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":564,"current":604} +{"time":"2026-05-05T06:21:35.844094649+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":565,"current":604} +{"time":"2026-05-05T06:21:35.844611133+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":565,"current":604} +{"time":"2026-05-05T06:21:35.844885508+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":565,"current":604} +{"time":"2026-05-05T06:21:36.249050972+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":565,"current":604} +{"time":"2026-05-05T06:21:36.256763984+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":565,"current":604} +{"time":"2026-05-05T06:21:36.257236918+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":565,"current":604} +{"time":"2026-05-05T06:22:22.177898205+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":566,"current":604} +{"time":"2026-05-05T06:22:22.178420995+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":566,"current":604} +{"time":"2026-05-05T06:22:22.178690681+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":566,"current":604} +{"time":"2026-05-05T06:22:23.732329657+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":566,"current":604} +{"time":"2026-05-05T06:22:23.736304409+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":566,"current":604} +{"time":"2026-05-05T06:22:23.736544546+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":566,"current":604} +{"time":"2026-05-05T06:23:08.126445825+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":567,"current":604} +{"time":"2026-05-05T06:23:08.126944622+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":567,"current":604} +{"time":"2026-05-05T06:23:08.127224367+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":567,"current":604} +{"time":"2026-05-05T06:23:09.712055276+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":567,"current":604} +{"time":"2026-05-05T06:23:09.715988069+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":567,"current":604} +{"time":"2026-05-05T06:23:09.716354625+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":567,"current":604} +{"time":"2026-05-05T06:23:54.057893348+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":568,"current":604} +{"time":"2026-05-05T06:23:54.058421744+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":568,"current":604} +{"time":"2026-05-05T06:23:54.058706799+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":568,"current":604} +{"time":"2026-05-05T06:23:55.6464045+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":568,"current":604} +{"time":"2026-05-05T06:23:55.650239683+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":568,"current":604} +{"time":"2026-05-05T06:23:55.650589316+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":568,"current":604} +{"time":"2026-05-05T06:24:37.527648579+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":569,"current":604} +{"time":"2026-05-05T06:24:37.5280887+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":569,"current":604} +{"time":"2026-05-05T06:24:37.528397025+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":569,"current":604} +{"time":"2026-05-05T06:24:39.101347101+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":569,"current":604} +{"time":"2026-05-05T06:24:39.105160557+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":569,"current":604} +{"time":"2026-05-05T06:24:39.105503202+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":569,"current":604} +{"time":"2026-05-05T06:27:24.32302829+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":570,"current":604} +{"time":"2026-05-05T06:27:24.323532015+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":570,"current":604} +{"time":"2026-05-05T06:27:24.323805534+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":570,"current":604} +{"time":"2026-05-05T06:27:24.726489071+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":570,"current":604} +{"time":"2026-05-05T06:27:24.734064243+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":570,"current":604} +{"time":"2026-05-05T06:27:24.734498465+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":570,"current":604} +{"time":"2026-05-05T06:28:13.325194524+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":571,"current":604} +{"time":"2026-05-05T06:28:13.325642495+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":571,"current":604} +{"time":"2026-05-05T06:28:13.325944037+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":571,"current":604} +{"time":"2026-05-05T06:28:14.894484121+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":571,"current":604} +{"time":"2026-05-05T06:28:14.898276838+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":571,"current":604} +{"time":"2026-05-05T06:28:14.898600627+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":571,"current":604} +{"time":"2026-05-05T06:28:56.822539966+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":572,"current":604} +{"time":"2026-05-05T06:28:56.823405893+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":572,"current":604} +{"time":"2026-05-05T06:28:56.823712559+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":572,"current":604} +{"time":"2026-05-05T06:28:58.389761723+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":572,"current":604} +{"time":"2026-05-05T06:28:58.393665171+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":572,"current":604} +{"time":"2026-05-05T06:28:58.394041293+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":572,"current":604} +{"time":"2026-05-05T06:29:42.706111099+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":573,"current":604} +{"time":"2026-05-05T06:29:42.706591348+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":573,"current":604} +{"time":"2026-05-05T06:29:42.706908371+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":573,"current":604} +{"time":"2026-05-05T06:29:44.278862086+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":573,"current":604} +{"time":"2026-05-05T06:29:44.282779643+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":573,"current":604} +{"time":"2026-05-05T06:29:44.283137774+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":573,"current":604} +{"time":"2026-05-05T06:30:28.616965903+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":574,"current":604} +{"time":"2026-05-05T06:30:28.617491931+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":574,"current":604} +{"time":"2026-05-05T06:30:28.617740639+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":574,"current":604} +{"time":"2026-05-05T06:30:30.188991621+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":574,"current":604} +{"time":"2026-05-05T06:30:30.192860194+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":574,"current":604} +{"time":"2026-05-05T06:30:30.193215709+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":574,"current":604} +{"time":"2026-05-05T06:33:15.131913145+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":575,"current":604} +{"time":"2026-05-05T06:33:15.132414888+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":575,"current":604} +{"time":"2026-05-05T06:33:15.132670194+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":575,"current":604} +{"time":"2026-05-05T06:33:15.542609089+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":575,"current":604} +{"time":"2026-05-05T06:33:15.550123084+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":575,"current":604} +{"time":"2026-05-05T06:33:15.550593066+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":575,"current":604} +{"time":"2026-05-05T06:34:01.650107093+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":576,"current":604} +{"time":"2026-05-05T06:34:01.650686231+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":576,"current":604} +{"time":"2026-05-05T06:34:01.650934063+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":576,"current":604} +{"time":"2026-05-05T06:34:03.217716254+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":576,"current":604} +{"time":"2026-05-05T06:34:03.221588948+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":576,"current":604} +{"time":"2026-05-05T06:34:03.221892622+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":576,"current":604} +{"time":"2026-05-05T06:34:47.623526892+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":577,"current":604} +{"time":"2026-05-05T06:34:47.624012153+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":577,"current":604} +{"time":"2026-05-05T06:34:47.624266627+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":577,"current":604} +{"time":"2026-05-05T06:34:49.181906496+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":577,"current":604} +{"time":"2026-05-05T06:34:49.185772356+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":577,"current":604} +{"time":"2026-05-05T06:34:49.186091579+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":577,"current":604} +{"time":"2026-05-05T06:35:30.999893977+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":578,"current":604} +{"time":"2026-05-05T06:35:31.000434186+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":578,"current":604} +{"time":"2026-05-05T06:35:31.000690691+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":578,"current":604} +{"time":"2026-05-05T06:35:32.570877186+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":578,"current":604} +{"time":"2026-05-05T06:35:32.574676325+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":578,"current":604} +{"time":"2026-05-05T06:35:32.575028066+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":578,"current":604} +{"time":"2026-05-05T06:36:16.848128158+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":579,"current":604} +{"time":"2026-05-05T06:36:16.848649411+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":579,"current":604} +{"time":"2026-05-05T06:36:16.84889655+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":579,"current":604} +{"time":"2026-05-05T06:36:18.424314964+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":579,"current":604} +{"time":"2026-05-05T06:36:18.428156089+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":579,"current":604} +{"time":"2026-05-05T06:36:18.428503896+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":579,"current":604} +{"time":"2026-05-05T06:38:58.128203352+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":580,"current":604} +{"time":"2026-05-05T06:38:58.128734288+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":580,"current":604} +{"time":"2026-05-05T06:38:58.128994491+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":580,"current":604} +{"time":"2026-05-05T06:38:58.526981035+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":580,"current":604} +{"time":"2026-05-05T06:38:58.534835738+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":580,"current":604} +{"time":"2026-05-05T06:38:58.535314647+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":580,"current":604} +{"time":"2026-05-05T06:39:44.504119651+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":581,"current":604} +{"time":"2026-05-05T06:39:44.50463747+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":581,"current":604} +{"time":"2026-05-05T06:39:44.50489251+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":581,"current":604} +{"time":"2026-05-05T06:39:46.079761867+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":581,"current":604} +{"time":"2026-05-05T06:39:46.083625673+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":581,"current":604} +{"time":"2026-05-05T06:39:46.083959063+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":581,"current":604} +{"time":"2026-05-05T06:40:27.834478364+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":582,"current":604} +{"time":"2026-05-05T06:40:27.835035292+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":582,"current":604} +{"time":"2026-05-05T06:40:27.835295725+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":582,"current":604} +{"time":"2026-05-05T06:40:29.40792299+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":582,"current":604} +{"time":"2026-05-05T06:40:29.411797366+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":582,"current":604} +{"time":"2026-05-05T06:40:29.41218707+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":582,"current":604} +{"time":"2026-05-05T06:41:13.793088164+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":583,"current":604} +{"time":"2026-05-05T06:41:13.79364781+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":583,"current":604} +{"time":"2026-05-05T06:41:13.7938797+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":583,"current":604} +{"time":"2026-05-05T06:41:15.369781282+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":583,"current":604} +{"time":"2026-05-05T06:41:15.373546731+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":583,"current":604} +{"time":"2026-05-05T06:41:15.37390131+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":583,"current":604} +{"time":"2026-05-05T06:41:59.920331994+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":584,"current":604} +{"time":"2026-05-05T06:41:59.920869489+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":584,"current":604} +{"time":"2026-05-05T06:41:59.921112097+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":584,"current":604} +{"time":"2026-05-05T06:42:01.496564606+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":584,"current":604} +{"time":"2026-05-05T06:42:01.500442427+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":584,"current":604} +{"time":"2026-05-05T06:42:01.500820769+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":584,"current":604} +{"time":"2026-05-05T06:44:49.178166639+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":585,"current":604} +{"time":"2026-05-05T06:44:49.178695961+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":585,"current":604} +{"time":"2026-05-05T06:44:49.178951102+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":585,"current":604} +{"time":"2026-05-05T06:44:49.579681074+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":585,"current":604} +{"time":"2026-05-05T06:44:49.587171777+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":585,"current":604} +{"time":"2026-05-05T06:44:49.587648184+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":585,"current":604} +{"time":"2026-05-05T06:45:35.733093296+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":586,"current":604} +{"time":"2026-05-05T06:45:35.733638724+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":586,"current":604} +{"time":"2026-05-05T06:45:35.733887211+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":586,"current":604} +{"time":"2026-05-05T06:45:37.305068171+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":586,"current":604} +{"time":"2026-05-05T06:45:37.309003004+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":586,"current":604} +{"time":"2026-05-05T06:45:37.309319787+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":586,"current":604} +{"time":"2026-05-05T06:46:24.347711074+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":587,"current":604} +{"time":"2026-05-05T06:46:24.348185707+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":587,"current":604} +{"time":"2026-05-05T06:46:24.348446316+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":587,"current":604} +{"time":"2026-05-05T06:46:25.923064122+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":587,"current":604} +{"time":"2026-05-05T06:46:25.926900027+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":587,"current":604} +{"time":"2026-05-05T06:46:25.927254931+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":587,"current":604} +{"time":"2026-05-05T06:47:10.238534002+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":588,"current":604} +{"time":"2026-05-05T06:47:10.239080279+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":588,"current":604} +{"time":"2026-05-05T06:47:10.239346768+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":588,"current":604} +{"time":"2026-05-05T06:47:11.807511887+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":588,"current":604} +{"time":"2026-05-05T06:47:11.811447+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":588,"current":604} +{"time":"2026-05-05T06:47:11.811790771+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":588,"current":604} +{"time":"2026-05-05T06:47:56.12314911+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":589,"current":604} +{"time":"2026-05-05T06:47:56.12365599+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":589,"current":604} +{"time":"2026-05-05T06:47:56.123923728+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":589,"current":604} +{"time":"2026-05-05T06:47:57.701912243+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":589,"current":604} +{"time":"2026-05-05T06:47:57.705849079+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":589,"current":604} +{"time":"2026-05-05T06:47:57.706168057+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":589,"current":604} +{"time":"2026-05-05T06:50:45.189821715+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":590,"current":604} +{"time":"2026-05-05T06:50:45.190339186+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":590,"current":604} +{"time":"2026-05-05T06:50:45.190580706+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":590,"current":604} +{"time":"2026-05-05T06:50:45.590388339+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":590,"current":604} +{"time":"2026-05-05T06:50:45.597702524+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":590,"current":604} +{"time":"2026-05-05T06:50:45.59819066+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":590,"current":604} +{"time":"2026-05-05T06:51:31.644141697+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":591,"current":604} +{"time":"2026-05-05T06:51:31.644682655+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":591,"current":604} +{"time":"2026-05-05T06:51:31.644911875+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":591,"current":604} +{"time":"2026-05-05T06:51:33.215471508+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":591,"current":604} +{"time":"2026-05-05T06:51:33.219353216+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":591,"current":604} +{"time":"2026-05-05T06:51:33.219617234+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":591,"current":604} +{"time":"2026-05-05T06:52:14.93242182+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":592,"current":604} +{"time":"2026-05-05T06:52:14.932954823+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":592,"current":604} +{"time":"2026-05-05T06:52:14.933220327+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":592,"current":604} +{"time":"2026-05-05T06:52:16.508741644+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":592,"current":604} +{"time":"2026-05-05T06:52:16.513057608+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":592,"current":604} +{"time":"2026-05-05T06:52:16.513401059+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":592,"current":604} +{"time":"2026-05-05T06:52:58.285828809+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":593,"current":604} +{"time":"2026-05-05T06:52:58.286363797+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":593,"current":604} +{"time":"2026-05-05T06:52:58.286623892+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":593,"current":604} +{"time":"2026-05-05T06:52:59.845228745+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":593,"current":604} +{"time":"2026-05-05T06:52:59.849068856+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":593,"current":604} +{"time":"2026-05-05T06:52:59.849426886+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":593,"current":604} +{"time":"2026-05-05T06:53:46.747882568+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":594,"current":604} +{"time":"2026-05-05T06:53:46.748399162+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":594,"current":604} +{"time":"2026-05-05T06:53:46.748627014+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":594,"current":604} +{"time":"2026-05-05T06:53:48.328465219+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":594,"current":604} +{"time":"2026-05-05T06:53:48.33229302+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":594,"current":604} +{"time":"2026-05-05T06:53:48.332641911+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":594,"current":604} +{"time":"2026-05-05T06:56:33.464590423+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":595,"current":604} +{"time":"2026-05-05T06:56:33.465072824+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":595,"current":604} +{"time":"2026-05-05T06:56:33.465353632+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":595,"current":604} +{"time":"2026-05-05T06:56:33.87053382+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":595,"current":604} +{"time":"2026-05-05T06:56:33.878070768+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":595,"current":604} +{"time":"2026-05-05T06:56:33.878561171+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":595,"current":604} +{"time":"2026-05-05T06:57:22.426878478+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":596,"current":604} +{"time":"2026-05-05T06:57:22.427469012+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":596,"current":604} +{"time":"2026-05-05T06:57:22.427749942+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":596,"current":604} +{"time":"2026-05-05T06:57:24.009015268+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":596,"current":604} +{"time":"2026-05-05T06:57:24.012936376+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":596,"current":604} +{"time":"2026-05-05T06:57:24.013290225+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":596,"current":604} +{"time":"2026-05-05T06:58:08.413978492+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":597,"current":604} +{"time":"2026-05-05T06:58:08.414450915+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":597,"current":604} +{"time":"2026-05-05T06:58:08.414727+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":597,"current":604} +{"time":"2026-05-05T06:58:10.011833293+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":597,"current":604} +{"time":"2026-05-05T06:58:10.015589444+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":597,"current":604} +{"time":"2026-05-05T06:58:10.015944333+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":597,"current":604} +{"time":"2026-05-05T06:58:51.785357544+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":598,"current":604} +{"time":"2026-05-05T06:58:51.785841788+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":598,"current":604} +{"time":"2026-05-05T06:58:51.786117341+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":598,"current":604} +{"time":"2026-05-05T06:58:53.365955255+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":598,"current":604} +{"time":"2026-05-05T06:58:53.369789928+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":598,"current":604} +{"time":"2026-05-05T06:58:53.370080219+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":598,"current":604} +{"time":"2026-05-05T06:59:40.464888076+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":599,"current":604} +{"time":"2026-05-05T06:59:40.465410272+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":599,"current":604} +{"time":"2026-05-05T06:59:40.465682214+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":599,"current":604} +{"time":"2026-05-05T06:59:42.04666856+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":599,"current":604} +{"time":"2026-05-05T06:59:42.050517435+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":599,"current":604} +{"time":"2026-05-05T06:59:42.050832671+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":599,"current":604} +{"time":"2026-05-05T07:02:21.861671255+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:02:21.862188665+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:02:21.862496192+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:02:22.258733643+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:02:22.266120477+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:02:22.266609566+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:02:39.912394613+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:03:48.566294784+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:03:48.566592429+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.568595282+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.569464197+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.569774187+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.57009833+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.570392368+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.570689838+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.570978213+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.57127051+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.571558985+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.571848481+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.572132884+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:04.572419878+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":600,"current":604} +{"time":"2026-05-05T07:04:45.488043735+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":604} +{"time":"2026-05-05T07:04:45.488579056+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":604} +{"time":"2026-05-05T07:04:45.488834735+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":604} +{"time":"2026-05-05T07:04:47.072651968+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":604} +{"time":"2026-05-05T07:04:47.076508177+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":604} +{"time":"2026-05-05T07:04:47.076844764+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":601,"current":604} +{"time":"2026-05-05T07:05:31.582773606+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":602,"current":604} +{"time":"2026-05-05T07:05:31.583365195+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":602,"current":604} +{"time":"2026-05-05T07:05:31.583621586+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":602,"current":604} +{"time":"2026-05-05T07:05:33.160295359+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":602,"current":604} +{"time":"2026-05-05T07:05:33.164124317+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":602,"current":604} +{"time":"2026-05-05T07:05:33.16448972+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":602,"current":604} +{"time":"2026-05-05T07:06:17.701049115+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":603,"current":604} +{"time":"2026-05-05T07:06:17.701553103+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":603,"current":604} +{"time":"2026-05-05T07:06:17.701846233+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":603,"current":604} +{"time":"2026-05-05T07:06:19.288291156+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":603,"current":604} +{"time":"2026-05-05T07:06:19.292307061+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":603,"current":604} +{"time":"2026-05-05T07:06:19.292650851+09:00","level":"WARN","msg":"handler: ignoring partial history record","step":603,"current":604} +{"time":"2026-05-05T12:57:09.545996172+09:00","level":"INFO","msg":"stream: closing","id":"lvwg9pfc"} +{"time":"2026-05-05T12:57:11.080356396+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T12:57:11.552499003+09:00","level":"INFO","msg":"handler: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T12:57:11.55258114+09:00","level":"INFO","msg":"sender: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T12:57:11.552597829+09:00","level":"INFO","msg":"stream: closed","id":"lvwg9pfc"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b3d530ba7cb0041e931a81a42a9b56142c81bd1b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug.log @@ -0,0 +1,22 @@ +2026-05-05 05:03:47,246 INFO MainThread:3400577 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 05:03:47,246 INFO MainThread:3400577 [wandb_setup.py:_flush():81] Configure stats pid to 3400577 +2026-05-05 05:03:47,246 INFO MainThread:3400577 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 05:03:47,246 INFO MainThread:3400577 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug.log +2026-05-05 05:03:47,246 INFO MainThread:3400577 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/logs/debug-internal.log +2026-05-05 05:03:47,246 INFO MainThread:3400577 [wandb_init.py:init():844] calling init triggers +2026-05-05 05:03:47,246 INFO MainThread:3400577 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '900', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 05:03:47,246 INFO MainThread:3400577 [wandb_init.py:init():892] starting backend +2026-05-05 05:03:47,479 INFO MainThread:3400577 [wandb_init.py:init():895] sending inform_init request +2026-05-05 05:03:47,492 INFO MainThread:3400577 [wandb_init.py:init():903] backend started and connected +2026-05-05 05:03:47,496 INFO MainThread:3400577 [wandb_init.py:init():973] updated telemetry +2026-05-05 05:03:47,516 INFO MainThread:3400577 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 05:03:51,321 INFO MainThread:3400577 [wandb_init.py:init():1037] run resumed +2026-05-05 05:03:51,325 INFO MainThread:3400577 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 05:03:51,592 INFO MainThread:3400577 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 05:03:51,592 INFO MainThread:3400577 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 05:03:51,592 INFO MainThread:3400577 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 05:03:51,592 INFO MainThread:3400577 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 05:03:51,596 INFO MainThread:3400577 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 12:57:09,545 INFO wandb-AsyncioManager-main:3400577 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 12:57:09,546 INFO wandb-AsyncioManager-main:3400577 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/run-lvwg9pfc.wandb b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/run-lvwg9pfc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f900c1f5f502d651b9dff7253d0bf24ea10f8339 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_050347-lvwg9pfc/run-lvwg9pfc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f9bbd6a971b8607fc2019cbc67c0d2a0505033676dd204c861d03461cd75a1a +size 6420140 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f430cd1752255398e6033ad65c4f3f24ddbc3e73 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/config.yaml @@ -0,0 +1,391 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + z2205v42lvq1qmd2kvnlslezx5dnogze: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py + - '-' + - trainer.resume=True + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_t769 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=1000 + - trainer.save_ckpt_iter=100 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1176050405376" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769 + startedAt: "2026-05-05T05:22:05.834554Z" + writerId: z2205v42lvq1qmd2kvnlslezx5dnogze + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "1000" + offload_module_in_decoding: "False" + resume: "True" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_042b02b2ceeb648cf655.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_042b02b2ceeb648cf655.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3caadf20fbb4e4b2b3b6808d12870b8d381eeeae --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_042b02b2ceeb648cf655.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:042b02b2ceeb648cf65516bc0efc2bf1fe36e19364265bff8c4de13fdac76ab9 +size 185990 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_1bd4125209338a80be7e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_1bd4125209338a80be7e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ed6a2aa12bb146694f84c4408b4b6deffc79ca16 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_1bd4125209338a80be7e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd4125209338a80be7e813303808aab3f1488b772b74d7029376a714b9687e9 +size 248914 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_1d5d847d7164bb28616f.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_1d5d847d7164bb28616f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23501929376395c21b2801ef0ed060f0103951f5 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_1d5d847d7164bb28616f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d5d847d7164bb28616fe152af9db791eb56fe5fb0ba93be150a376f5f7e9d08 +size 208735 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_4b40b35b2d64012475c8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_4b40b35b2d64012475c8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52d632fa0fe5631d9fb8f5170cfb97c9bcfdef5d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_4b40b35b2d64012475c8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b40b35b2d64012475c8b8e1a88fbf5e8a6c8ff6b45f0e3d3867c7f945bc7710 +size 207550 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7105cc2283e6acf8678a.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7105cc2283e6acf8678a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a18f333c55800247b00916b5847f25eb92a30edb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7105cc2283e6acf8678a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7105cc2283e6acf8678a347bea69119f41eaef4741e3561e0b0967674de01b45 +size 299641 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_776855bea09e3ee28d4f.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_776855bea09e3ee28d4f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6039734230675bd0cf8ea7884dffed773a3e3266 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_776855bea09e3ee28d4f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:776855bea09e3ee28d4faca7baa99c6096a321e6de248d89d5055f4f0b34b34e +size 222921 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7a8d3fc9242239c85404.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7a8d3fc9242239c85404.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d590b6e2b360b0b42f51c4cf71da45fd24c66ed6 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7a8d3fc9242239c85404.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a8d3fc9242239c85404ca99373c3cfbb5c154df530b2dd222c65e8acfc75a3a +size 261519 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7f1688b26c115ba620a5.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7f1688b26c115ba620a5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6e33ec5207613436a8d3962650447a3dbde5b4b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_7f1688b26c115ba620a5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f1688b26c115ba620a57db2534d1d09c5dc04798c86d4b94a3379504649a26a +size 218000 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_cdda7c097236ecac61ec.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_cdda7c097236ecac61ec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..809f4ac2d4ef508c266e62f4f7c1a12d6b682282 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_cdda7c097236ecac61ec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdda7c097236ecac61ec1ffd5061031d0134fad2bfe519fcc070ed7db02c2972 +size 241147 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_e55b0b718380e13aab7d.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_e55b0b718380e13aab7d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24291f41f815b34b2f2ac7ef6f245df8995b4aaf --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/generated_1000_e55b0b718380e13aab7d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e55b0b718380e13aab7d29c90c6c66173eddf8b51a2d1ffc2c61e629d0212dc7 +size 304501 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/media/videos/val0/reconstructed_1000_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/output.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0c1c0bb46b52a8f304d1bb5db8f5e84316415d21 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/output.log @@ -0,0 +1,2094 @@ +[May 5, 2026 - 14:22:07 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 5, 2026 - 14:22:07 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 5, 2026 - 14:22:07 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 5, 2026 - 14:22:07 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 5, 2026 - 14:22:07 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 5, 2026 - 14:22:07 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 5, 2026 - 14:22:07 | INFO | fastgen.trainer:run:77 ] Starting training +[May 5, 2026 - 14:22:07 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 5, 2026 - 14:22:07 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 5, 2026 - 14:22:35 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 5, 2026 - 14:22:47 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 5, 2026 - 14:22:47 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 5, 2026 - 14:22:47 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 5, 2026 - 14:22:47 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 5, 2026 - 14:23:31 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 5, 2026 - 14:23:31 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 5, 2026 - 14:23:31 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 5, 2026 - 14:23:31 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 5, 2026 - 14:23:31 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 5, 2026 - 14:23:31 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 5, 2026 - 14:23:33 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 14:23:33 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 5, 2026 - 14:23:33 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 5, 2026 - 14:23:33 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 14:23:35 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 14:23:35 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.9s +[May 5, 2026 - 14:23:35 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 5, 2026 - 14:23:35 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 14:23:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 14:23:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.2s +[May 5, 2026 - 14:23:47 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 5, 2026 - 14:23:47 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 5, 2026 - 14:23:47 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 5, 2026 - 14:23:47 | INFO | fastgen.trainer:run:137 ] Loading checkpoints for resuming .. +[May 5, 2026 - 14:23:47 | INFO | fastgen.utils.checkpointer:load:436 ] Loading FSDP model from prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000900 +[May 5, 2026 - 14:23:47 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key net... +[May 5, 2026 - 14:23:50 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key fake_score... +[May 5, 2026 - 14:23:50 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key net... +[May 5, 2026 - 14:23:50 | SUCCESS | fastgen.utils.checkpointer:load:544 ] Successfully loaded optimizer state for net +[May 5, 2026 - 14:23:50 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key fake_score... +[May 5, 2026 - 14:23:51 | SUCCESS | fastgen.utils.checkpointer:load:544 ] Successfully loaded optimizer state for fake_score +[May 5, 2026 - 14:23:51 | INFO | fastgen.utils.checkpointer:load:565 ] Loading the scheduler_dict... +[May 5, 2026 - 14:23:51 | INFO | fastgen.utils.checkpointer:load:573 ] Loading the gradient scaler... +[May 5, 2026 - 14:23:51 | WARNING | fastgen.utils.checkpointer:load:578 ] Gradient scaler state is empty (likely saved from disabled scaler), skipping load. +[May 5, 2026 - 14:23:51 | INFO | fastgen.utils.checkpointer:load:581 ] Loading the callbacks... +[May 5, 2026 - 14:23:51 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 900. +[May 5, 2026 - 14:23:51 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17150.48 M total params (logical). +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4299.19 M total params LOCAL on rank 0. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 1. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 2. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 3. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 14:23:57 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 14:23:57 | INFO | fastgen.trainer:run:174 ] iter_start: 900 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.440848 37.571041 37.391792 +peak_gpu_mem_gb 58.406074 58.407636 58.405553 +peak_gpu_mem_reserved_gb 60.617676 60.619141 60.617188 +util 79.750000 85.000000 71.000000 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 901 : data loading time 1.29 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 901 : avg forward pass time 11.69 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 901 : backward pass time 10.14 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 901 : optimizer step time 1.76 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 901-------------------- +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:24:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.616031 37.747494 37.566608 +peak_gpu_mem_gb 61.054405 61.056748 61.053624 +peak_gpu_mem_reserved_gb 62.397461 62.400391 62.396484 +util 92.500000 97.000000 90.000000 +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 902 : avg iteration time 46.11 seconds +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 902 : data loading time 0.00 +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 902 : avg forward pass time 12.10 +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 902 : backward pass time 10.17 +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 902 : optimizer step time 1.57 +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 902-------------------- +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:25:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.616318 37.747803 37.566902 +peak_gpu_mem_gb 61.054405 61.056748 61.053624 +peak_gpu_mem_reserved_gb 62.338867 62.341797 62.337891 +util 96.000000 99.000000 94.000000 +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 903 : avg iteration time 48.64 seconds +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 903 : data loading time 0.00 +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 903 : avg forward pass time 13.36 +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 903 : backward pass time 10.16 +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 903 : optimizer step time 1.59 +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 903-------------------- +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:26:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.616702 37.747944 37.567059 +peak_gpu_mem_gb 61.054405 61.056748 61.053624 +peak_gpu_mem_reserved_gb 62.319336 62.322266 62.318359 +util 90.500000 95.000000 88.000000 +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 904 : avg iteration time 45.95 seconds +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 904 : data loading time 0.00 +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 904 : avg forward pass time 12.04 +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 904 : backward pass time 10.13 +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 904 : optimizer step time 1.58 +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 904-------------------- +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:27:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.676462 37.806728 37.628052 +peak_gpu_mem_gb 127.092922 127.095265 127.092141 +peak_gpu_mem_reserved_gb 131.028320 131.031250 131.027344 +util 60.000000 65.000000 52.000000 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 905 : avg iteration time 175.33 seconds +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 905 : data loading time 0.00 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 905 : avg forward pass time 78.94 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 905 : backward pass time 8.47 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 905 : optimizer step time 0.44 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 905-------------------- +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 48217.5928 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4886 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 48217.5928iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4399iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4482 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.7764 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 107691.0859iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 18.1088iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 769015.9375iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4886iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2372 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 35394.7773iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.5169 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 8381.8428iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.7764 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 769015.9375iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.2149 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 83.9137 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:30:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.676688 37.807194 37.628216 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.750977 64.753906 64.750000 +util 91.500000 96.000000 89.000000 +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 906 : avg iteration time 47.68 seconds +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 906 : data loading time 0.00 +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 906 : avg forward pass time 12.90 +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 906 : backward pass time 10.14 +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 906 : optimizer step time 1.58 +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 906-------------------- +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:30:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.677202 37.807495 37.628483 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.726562 64.730469 64.714844 +util 93.500000 98.000000 91.000000 +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 907 : avg iteration time 48.63 seconds +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 907 : data loading time 0.00 +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 907 : avg forward pass time 13.38 +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 907 : backward pass time 10.14 +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 907 : optimizer step time 1.57 +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 907-------------------- +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.677442 37.807999 37.628712 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.980469 65.062500 64.734375 +util 95.750000 99.000000 88.000000 +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 908 : avg iteration time 43.28 seconds +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 908 : data loading time 0.00 +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 908 : avg forward pass time 10.72 +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 908 : backward pass time 10.12 +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 908 : optimizer step time 1.58 +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 908-------------------- +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:32:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.677708 37.808472 37.628948 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.731445 64.734375 64.730469 +util 98.250000 100.000000 97.000000 +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 909 : avg iteration time 48.58 seconds +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 909 : data loading time 0.00 +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 909 : avg forward pass time 13.36 +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 909 : backward pass time 10.13 +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 909 : optimizer step time 1.58 +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 909-------------------- +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:33:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.679324 37.809063 37.630711 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.628418 133.632812 133.626953 +util 67.250000 72.000000 64.000000 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 910 : avg iteration time 163.20 seconds +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 910 : data loading time 0.00 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 910 : avg forward pass time 72.95 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 910 : backward pass time 8.43 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 910 : optimizer step time 0.40 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 910-------------------- +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 122017.6514 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5194 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 122017.6514iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.2895iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4717 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5285 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 274316.2188iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 18.9785iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3460576.7500iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5194iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.1694 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1687651.0000iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2959 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5387.0898iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2120 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 33654.9648iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.9218 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 139185.5312iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:35:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 14:36:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.679729 37.809887 37.631142 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.912109 65.046875 64.867188 +util 92.500000 97.000000 90.000000 +[May 5, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 911 : avg iteration time 50.32 seconds +[May 5, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 911 : data loading time 0.00 +[May 5, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 911 : avg forward pass time 14.24 +[May 5, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 911 : backward pass time 10.14 +[May 5, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 911 : optimizer step time 1.58 +[May 5, 2026 - 14:36:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 911-------------------- +[May 5, 2026 - 14:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0147 iter count: 1.0 +[May 5, 2026 - 14:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0147 iter count: 1.0 +[May 5, 2026 - 14:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:36:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.679814 37.809940 37.631245 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.863281 65.085938 64.789062 +util 92.000000 99.000000 87.000000 +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 912 : avg iteration time 43.36 seconds +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 912 : data loading time 0.00 +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 912 : avg forward pass time 10.76 +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 912 : backward pass time 10.12 +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 912 : optimizer step time 1.57 +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 912-------------------- +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:37:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.680010 37.810238 37.631523 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.858398 65.066406 64.789062 +util 95.500000 100.000000 89.000000 +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 913 : avg iteration time 48.59 seconds +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 913 : data loading time 0.00 +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 913 : avg forward pass time 13.36 +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 913 : backward pass time 10.17 +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 913 : optimizer step time 1.56 +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 913-------------------- +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:38:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.680375 37.811253 37.631504 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.858398 65.066406 64.789062 +util 95.250000 100.000000 89.000000 +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 914 : avg iteration time 46.00 seconds +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 914 : data loading time 0.00 +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 914 : avg forward pass time 12.07 +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 914 : backward pass time 10.15 +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 914 : optimizer step time 1.57 +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 914-------------------- +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:39:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.680606 37.810974 37.631977 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.663574 133.667969 133.650391 +util 66.000000 71.000000 62.000000 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 915 : avg iteration time 165.90 seconds +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 915 : data loading time 0.00 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 915 : avg forward pass time 74.36 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 915 : backward pass time 8.41 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 915 : optimizer step time 0.40 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 915-------------------- +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 388681.9568 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4899 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 388681.9568iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7455iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7504 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1526 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 749577.8750iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.4855 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 12058343.0000iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4899iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.0133 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 167144.3750iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5484 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 65959.5547iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7988 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1993.5846iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6640 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1522.4344iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:41:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.680720 37.811253 37.631977 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.034180 65.042969 65.007812 +util 91.500000 96.000000 89.000000 +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 916 : avg iteration time 47.75 seconds +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 916 : data loading time 0.00 +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 916 : avg forward pass time 12.94 +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 916 : backward pass time 10.14 +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 916 : optimizer step time 1.60 +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 916-------------------- +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:42:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.680932 37.811951 37.632008 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.048828 65.066406 65.042969 +util 92.250000 99.000000 86.000000 +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 917 : avg iteration time 43.35 seconds +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 917 : data loading time 0.00 +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 917 : avg forward pass time 10.73 +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 917 : backward pass time 10.14 +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 917 : optimizer step time 1.57 +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 917-------------------- +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:43:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.681285 37.812317 37.633045 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.039062 65.042969 65.027344 +util 94.500000 99.000000 92.000000 +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 918 : avg iteration time 43.46 seconds +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 918 : data loading time 0.00 +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 918 : avg forward pass time 10.80 +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 918 : backward pass time 10.14 +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 918 : optimizer step time 1.57 +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 918-------------------- +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0138 iter count: 1.0 +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:44:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.681200 37.811718 37.633045 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.058594 65.105469 65.042969 +util 97.250000 100.000000 90.000000 +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 919 : avg iteration time 48.69 seconds +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 919 : data loading time 0.00 +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 919 : avg forward pass time 13.40 +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 919 : backward pass time 10.15 +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 919 : optimizer step time 1.57 +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 919-------------------- +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:44:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.681253 37.811306 37.633343 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.618652 133.652344 133.607422 +util 61.000000 66.000000 58.000000 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 920 : avg iteration time 165.69 seconds +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 920 : data loading time 0.00 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 920 : avg forward pass time 74.27 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 920 : backward pass time 8.40 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 920 : optimizer step time 0.40 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 920-------------------- +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 261078.4199 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5327 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 261078.4199iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7514iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.1667 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0007 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 526702.6875iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 76.1966iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8898289.0000iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5327iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2895 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 290393.1250iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.8653 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 16825.7734iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.7898 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 789895.2500iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 8.0007 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8898289.0000iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:47:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.681372 37.811340 37.633545 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.912109 65.046875 64.867188 +util 92.000000 97.000000 89.000000 +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 921 : avg iteration time 47.60 seconds +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 921 : data loading time 0.00 +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 921 : avg forward pass time 12.87 +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 921 : backward pass time 10.15 +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 921 : optimizer step time 1.56 +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 921-------------------- +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.681801 37.812832 37.633671 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.877930 64.886719 64.851562 +util 94.750000 100.000000 89.000000 +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 922 : avg iteration time 45.94 seconds +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 922 : data loading time 0.00 +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 922 : avg forward pass time 12.04 +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 922 : backward pass time 10.14 +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 922 : optimizer step time 1.58 +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 922-------------------- +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:49:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682059 37.813255 37.634052 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.829102 64.832031 64.828125 +util 97.000000 100.000000 95.000000 +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 923 : avg iteration time 48.72 seconds +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 923 : data loading time 0.00 +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 923 : avg forward pass time 13.43 +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 923 : backward pass time 10.14 +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 923 : optimizer step time 1.58 +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 923-------------------- +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:49:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.681972 37.812565 37.634041 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.941406 64.984375 64.812500 +util 95.250000 100.000000 89.000000 +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 924 : avg iteration time 48.70 seconds +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 924 : data loading time 0.00 +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 924 : avg forward pass time 13.42 +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 924 : backward pass time 10.14 +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 924 : optimizer step time 1.57 +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 924-------------------- +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:50:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.681897 37.812130 37.634075 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.613770 133.632812 133.607422 +util 68.250000 74.000000 60.000000 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 925 : avg iteration time 170.98 seconds +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 925 : data loading time 0.00 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 925 : avg forward pass time 76.89 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 925 : backward pass time 8.42 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 925 : optimizer step time 0.40 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 925-------------------- +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 163688.7012 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5109 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 163688.7012iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3779iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7979 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4254 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 311894.0000iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.9323 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2815874.2500iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5109iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.8906 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 324.1771iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.0147 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 7.6095 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3506 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 44412.1016iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.7900 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 790168.0000iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:53:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682016 37.812115 37.634193 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.804688 65.144531 64.691406 +util 93.500000 98.000000 91.000000 +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 926 : avg iteration time 47.68 seconds +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 926 : data loading time 0.00 +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 926 : avg forward pass time 12.90 +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 926 : backward pass time 10.14 +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 926 : optimizer step time 1.58 +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 926-------------------- +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0149 iter count: 1.0 +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:54:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682356 37.813129 37.634239 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.829102 64.890625 64.808594 +util 95.750000 99.000000 88.000000 +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 927 : avg iteration time 45.88 seconds +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 927 : data loading time 0.00 +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 927 : avg forward pass time 12.02 +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 927 : backward pass time 10.14 +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 927 : optimizer step time 1.57 +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 927-------------------- +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:55:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682453 37.813339 37.634350 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.750977 64.812500 64.730469 +util 97.750000 100.000000 96.000000 +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 928 : avg iteration time 48.58 seconds +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 928 : data loading time 0.00 +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 928 : avg forward pass time 13.35 +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 928 : backward pass time 10.14 +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 928 : optimizer step time 1.57 +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 928-------------------- +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682317 37.812798 37.634369 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.824219 64.828125 64.812500 +util 96.000000 99.000000 94.000000 +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 929 : avg iteration time 43.38 seconds +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 929 : data loading time 0.00 +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 929 : avg forward pass time 10.76 +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 929 : backward pass time 10.14 +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 929 : optimizer step time 1.59 +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 929-------------------- +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:56:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682323 37.811680 37.635330 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.671875 133.689453 133.666016 +util 68.500000 74.000000 61.000000 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 930 : avg iteration time 163.09 seconds +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 930 : data loading time 0.00 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 930 : avg forward pass time 72.95 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 930 : backward pass time 8.41 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 930 : optimizer step time 0.40 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 930-------------------- +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1669896.5610 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4935 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1669896.5610iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5712iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9412 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.1780 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 3128687.7500iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 48.5368iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 93738360.0000iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4935iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.0073 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 22348.6797iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3229 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 42017.5078iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4499 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 992.0283iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.7772 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 104226.6641iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 14:59:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682383 37.811768 37.635456 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.882812 64.929688 64.867188 +util 94.250000 99.000000 91.000000 +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 931 : avg iteration time 44.88 seconds +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 931 : data loading time 0.00 +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 931 : avg forward pass time 11.52 +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 931 : backward pass time 10.15 +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 931 : optimizer step time 1.56 +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 931-------------------- +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:00:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682583 37.812553 37.635468 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.750977 64.808594 64.578125 +util 94.750000 99.000000 92.000000 +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 932 : avg iteration time 48.61 seconds +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 932 : data loading time 0.00 +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 932 : avg forward pass time 13.35 +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 932 : backward pass time 10.15 +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 932 : optimizer step time 1.58 +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 932-------------------- +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:01:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682601 37.812531 37.635487 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.892578 64.910156 64.886719 +util 97.000000 99.000000 91.000000 +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 933 : avg iteration time 43.30 seconds +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 933 : data loading time 0.00 +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 933 : avg forward pass time 10.71 +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 933 : backward pass time 10.14 +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 933 : optimizer step time 1.58 +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 933-------------------- +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:01:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682633 37.812531 37.635487 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.912109 64.988281 64.886719 +util 96.750000 100.000000 94.000000 +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 934 : avg iteration time 43.33 seconds +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 934 : data loading time 0.00 +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 934 : avg forward pass time 10.73 +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 934 : backward pass time 10.14 +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 934 : optimizer step time 1.59 +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 934-------------------- +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:02:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682279 37.810677 37.635700 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.632812 133.650391 133.626953 +util 59.500000 64.000000 57.000000 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 935 : avg iteration time 169.31 seconds +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 935 : data loading time 0.00 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 935 : avg forward pass time 76.04 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 935 : backward pass time 8.42 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 935 : optimizer step time 0.40 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 935-------------------- +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 351759.4482 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5092 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 351759.4482iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7554iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8561 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9763 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 667089.2500iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.5408 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8475095.0000iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5092iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.4151 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 373341.1562iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.4276 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 17.3770 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.7989 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 108864.4141iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.9394 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 357.3717iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:05:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682468 37.811188 37.635700 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.799805 65.007812 64.730469 +util 97.250000 100.000000 91.000000 +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 936 : avg iteration time 47.55 seconds +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 936 : data loading time 0.00 +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 936 : avg forward pass time 12.84 +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 936 : backward pass time 10.13 +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 936 : optimizer step time 1.59 +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 936-------------------- +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0143 iter count: 1.0 +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:06:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682764 37.812157 37.635700 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.809570 65.046875 64.730469 +util 92.500000 97.000000 90.000000 +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 937 : avg iteration time 48.68 seconds +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 937 : data loading time 0.00 +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 937 : avg forward pass time 13.41 +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 937 : backward pass time 10.14 +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 937 : optimizer step time 1.57 +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 937-------------------- +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:06:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682820 37.812157 37.635731 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.790039 65.027344 64.710938 +util 93.750000 99.000000 91.000000 +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 938 : avg iteration time 48.41 seconds +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 938 : data loading time 0.00 +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 938 : avg forward pass time 13.31 +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 938 : backward pass time 10.12 +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 938 : optimizer step time 1.56 +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 938-------------------- +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:07:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682875 37.812263 37.635731 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.804688 65.027344 64.730469 +util 94.750000 100.000000 88.000000 +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 939 : avg iteration time 48.53 seconds +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 939 : data loading time 0.00 +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 939 : avg forward pass time 13.34 +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 939 : backward pass time 10.13 +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 939 : optimizer step time 1.58 +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 939-------------------- +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:08:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682509 37.810585 37.635830 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.647949 133.666016 133.593750 +util 67.750000 73.000000 61.000000 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 940 : avg iteration time 162.98 seconds +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 940 : data loading time 0.00 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 940 : avg forward pass time 72.87 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 940 : backward pass time 8.43 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 940 : optimizer step time 0.40 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 940-------------------- +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 254598.0640 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4308 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 254598.0640iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8868iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2426 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2937 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 832951.2500iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.0027iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 15987755.0000iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4308iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.6373 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 10663.9160iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.8427 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 16081.8672iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4435 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 979.4111iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2035 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 33090.9414iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:11:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682689 37.811153 37.635838 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.034180 65.062500 64.949219 +util 96.750000 99.000000 90.000000 +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 941 : avg iteration time 44.99 seconds +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 941 : data loading time 0.00 +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 941 : avg forward pass time 11.56 +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 941 : backward pass time 10.14 +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 941 : optimizer step time 1.58 +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 941-------------------- +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:11:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683098 37.812645 37.635838 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.985352 65.003906 64.929688 +util 95.500000 99.000000 90.000000 +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 942 : avg iteration time 48.44 seconds +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 942 : data loading time 0.00 +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 942 : avg forward pass time 13.30 +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 942 : backward pass time 10.13 +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 942 : optimizer step time 1.57 +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 942-------------------- +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:12:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683116 37.812656 37.635841 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.780273 64.929688 64.730469 +util 90.000000 95.000000 87.000000 +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 943 : avg iteration time 46.01 seconds +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 943 : data loading time 0.00 +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 943 : avg forward pass time 12.07 +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 943 : backward pass time 10.16 +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 943 : optimizer step time 1.56 +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 943-------------------- +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0150 iter count: 1.0 +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:13:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683270 37.813175 37.635948 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.721680 64.929688 64.652344 +util 93.250000 98.000000 90.000000 +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 944 : avg iteration time 46.04 seconds +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 944 : data loading time 0.00 +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 944 : avg forward pass time 12.06 +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 944 : backward pass time 10.14 +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 944 : optimizer step time 1.58 +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 944-------------------- +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:14:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682787 37.811069 37.635975 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.623535 133.626953 133.613281 +util 61.500000 66.000000 58.000000 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 945 : avg iteration time 170.99 seconds +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 945 : data loading time 0.00 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 945 : avg forward pass time 76.90 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 945 : backward pass time 8.41 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 945 : optimizer step time 0.40 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 945-------------------- +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 618700.7109 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4717 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 618700.7109iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.6339iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.5005 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0662 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1237589.5000iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 148.5587iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10144715.0000iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4717iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.2600 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2023011.5000iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.8357 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 290.4226iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.5953 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 535306.0000iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.6248 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 4195728.5000iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:17:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682814 37.811081 37.636044 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.804688 64.851562 64.789062 +util 96.000000 99.000000 94.000000 +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 946 : avg iteration time 45.05 seconds +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 946 : data loading time 0.00 +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 946 : avg forward pass time 11.60 +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 946 : backward pass time 10.12 +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 946 : optimizer step time 1.57 +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 946-------------------- +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0161 iter count: 1.0 +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0161 iter count: 1.0 +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:17:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683191 37.812588 37.636021 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.687500 64.734375 64.671875 +util 94.000000 99.000000 91.000000 +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 947 : avg iteration time 43.37 seconds +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 947 : data loading time 0.00 +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 947 : avg forward pass time 10.75 +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 947 : backward pass time 10.14 +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 947 : optimizer step time 1.57 +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 947-------------------- +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:18:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683178 37.812592 37.635998 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.780273 64.808594 64.695312 +util 95.250000 100.000000 89.000000 +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 948 : avg iteration time 46.13 seconds +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 948 : data loading time 0.00 +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 948 : avg forward pass time 12.12 +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 948 : backward pass time 10.16 +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 948 : optimizer step time 1.58 +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 948-------------------- +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:19:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683178 37.812569 37.636044 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 64.808594 64.675781 +util 95.500000 100.000000 89.000000 +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 949 : avg iteration time 46.01 seconds +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 949 : data loading time 0.00 +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 949 : avg forward pass time 12.06 +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 949 : backward pass time 10.14 +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 949 : optimizer step time 1.59 +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 949-------------------- +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:20:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682797 37.811050 37.635998 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.637695 133.646484 133.611328 +util 66.500000 71.000000 63.000000 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 950 : avg iteration time 160.91 seconds +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 950 : data loading time 0.00 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 950 : avg forward pass time 71.87 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 950 : backward pass time 8.43 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 950 : optimizer step time 0.40 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 950-------------------- +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 164028.5942 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4455 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 164028.5942iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8892iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4308 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5432 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 370459.3750iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 17.4893iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3563810.5000iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4455iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.2516 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1988937.0000iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.2882 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 718.0095iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.5716 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 171.2747iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.3884 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 877.2337iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:22:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682858 37.811073 37.636150 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 64.789062 64.734375 +util 94.000000 99.000000 91.000000 +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 951 : avg iteration time 50.14 seconds +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 951 : data loading time 0.00 +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 951 : avg forward pass time 14.15 +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 951 : backward pass time 10.14 +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 951 : optimizer step time 1.57 +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 951-------------------- +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:23:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683163 37.812302 37.636154 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.814453 65.183594 64.691406 +util 94.500000 100.000000 91.000000 +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 952 : avg iteration time 46.00 seconds +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 952 : data loading time 0.00 +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 952 : avg forward pass time 12.06 +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 952 : backward pass time 10.16 +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 952 : optimizer step time 1.57 +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 952-------------------- +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:24:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683296 37.812660 37.636166 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.697266 64.714844 64.691406 +util 97.250000 99.000000 95.000000 +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 953 : avg iteration time 43.24 seconds +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 953 : data loading time 0.00 +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 953 : avg forward pass time 10.69 +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 953 : backward pass time 10.14 +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 953 : optimizer step time 1.58 +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 953-------------------- +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:25:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683346 37.812660 37.636185 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.819336 65.203125 64.691406 +util 90.500000 96.000000 87.000000 +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 954 : avg iteration time 46.10 seconds +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 954 : data loading time 0.00 +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 954 : avg forward pass time 12.13 +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 954 : backward pass time 10.14 +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 954 : optimizer step time 1.56 +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 954-------------------- +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:25:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682986 37.811199 37.636177 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.691406 133.705078 133.650391 +util 59.750000 65.000000 56.000000 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 955 : avg iteration time 166.00 seconds +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 955 : data loading time 0.00 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 955 : avg forward pass time 74.40 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 955 : backward pass time 8.44 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 955 : optimizer step time 0.40 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 955-------------------- +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 89421.4604 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4390 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 89421.4604iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8463iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9328 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5582 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 252031.1719iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 47.7348iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3672739.5000iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4390iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.5582 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3672739.5000iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.6667 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 207.1519iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.8084 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 15013.9277iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4256 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 6983.1968iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:28:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.682989 37.811199 37.636192 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.951172 65.003906 64.792969 +util 95.750000 99.000000 88.000000 +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 956 : avg iteration time 44.91 seconds +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 956 : data loading time 0.00 +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 956 : avg forward pass time 11.52 +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 956 : backward pass time 10.14 +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 956 : optimizer step time 1.58 +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 956-------------------- +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0145 iter count: 1.0 +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:29:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683418 37.812939 37.636192 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.029297 65.082031 64.871094 +util 92.500000 99.000000 87.000000 +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 957 : avg iteration time 45.87 seconds +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 957 : data loading time 0.00 +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 957 : avg forward pass time 11.99 +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 957 : backward pass time 10.16 +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 957 : optimizer step time 1.58 +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 957-------------------- +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:30:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683726 37.813934 37.636192 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.985352 65.003906 64.929688 +util 94.500000 99.000000 92.000000 +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 958 : avg iteration time 43.46 seconds +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 958 : data loading time 0.00 +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 958 : avg forward pass time 10.81 +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 958 : backward pass time 10.13 +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 958 : optimizer step time 1.56 +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 958-------------------- +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0138 iter count: 1.0 +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:30:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683824 37.814285 37.636196 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.897461 64.988281 64.867188 +util 92.250000 97.000000 89.000000 +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 959 : avg iteration time 43.36 seconds +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 959 : data loading time 0.00 +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 959 : avg forward pass time 10.76 +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 959 : backward pass time 10.14 +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 959 : optimizer step time 1.57 +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 959-------------------- +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:31:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683178 37.811680 37.636211 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.623047 133.626953 133.611328 +util 64.000000 68.000000 61.000000 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 960 : avg iteration time 166.04 seconds +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 960 : data loading time 0.00 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 960 : avg forward pass time 74.40 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 960 : backward pass time 8.46 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 960 : optimizer step time 0.40 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 960-------------------- +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 30659.9312 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5192 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 30659.9312iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4626iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6173 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.7102 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 59849.1797iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 25.3966iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 673666.3750iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5192iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.1536 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4053.0706iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9168 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 18649.6719iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7324 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1745.4062iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.7469 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 13276.9404iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:34:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683189 37.811691 37.636223 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.907227 65.027344 64.867188 +util 94.250000 99.000000 91.000000 +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 961 : avg iteration time 50.18 seconds +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 961 : data loading time 0.00 +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 961 : avg forward pass time 14.15 +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 961 : backward pass time 10.13 +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 961 : optimizer step time 1.58 +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 961-------------------- +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0151 iter count: 1.0 +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0151 iter count: 1.0 +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:35:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683578 37.813183 37.636253 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.794922 65.046875 64.710938 +util 95.250000 100.000000 88.000000 +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 962 : avg iteration time 46.06 seconds +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 962 : data loading time 0.00 +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 962 : avg forward pass time 12.10 +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 962 : backward pass time 10.15 +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 962 : optimizer step time 1.58 +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 962-------------------- +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:36:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683798 37.813953 37.636333 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.858398 65.027344 64.710938 +util 96.000000 100.000000 93.000000 +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 963 : avg iteration time 45.95 seconds +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 963 : data loading time 0.00 +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 963 : avg forward pass time 12.04 +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 963 : backward pass time 10.15 +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 963 : optimizer step time 1.56 +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 963-------------------- +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:36:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683924 37.814434 37.636360 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.853516 65.046875 64.710938 +util 91.000000 96.000000 88.000000 +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 964 : avg iteration time 43.54 seconds +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 964 : data loading time 0.00 +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 964 : avg forward pass time 10.83 +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 964 : backward pass time 10.16 +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 964 : optimizer step time 1.57 +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 964-------------------- +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0148 iter count: 1.0 +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:37:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683291 37.811848 37.636368 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.649414 133.652344 133.648438 +util 59.000000 64.000000 55.000000 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 965 : avg iteration time 166.00 seconds +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 965 : data loading time 0.00 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 965 : avg forward pass time 74.38 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 965 : backward pass time 8.43 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 965 : optimizer step time 0.40 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 965-------------------- +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 851014.3652 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4687 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 851014.3652iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8999iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6624 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5221 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1544369.8750iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 27.7910iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 25246958.0000iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4687iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6343 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1434.5386iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3685 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 46024.1836iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2570 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4983.9355iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.7381 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1765.6641iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:40:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683319 37.811893 37.636383 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.965820 65.101562 64.792969 +util 96.000000 100.000000 93.000000 +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 966 : avg iteration time 47.66 seconds +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 966 : data loading time 0.00 +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 966 : avg forward pass time 12.90 +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 966 : backward pass time 10.14 +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 966 : optimizer step time 1.58 +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 966-------------------- +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:41:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683796 37.813709 37.636383 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.853516 64.925781 64.773438 +util 92.250000 97.000000 89.000000 +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 967 : avg iteration time 48.75 seconds +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 967 : data loading time 0.00 +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 967 : avg forward pass time 13.43 +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 967 : backward pass time 10.15 +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 967 : optimizer step time 1.59 +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 967-------------------- +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:41:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683815 37.813763 37.636406 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.004883 65.222656 64.789062 +util 92.750000 98.000000 89.000000 +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 968 : avg iteration time 43.41 seconds +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 968 : data loading time 0.00 +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 968 : avg forward pass time 10.78 +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 968 : backward pass time 10.14 +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 968 : optimizer step time 1.56 +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 968-------------------- +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:42:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683825 37.813744 37.636410 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.029297 65.242188 64.789062 +util 96.500000 100.000000 89.000000 +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 969 : avg iteration time 45.90 seconds +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 969 : data loading time 0.00 +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 969 : avg forward pass time 12.02 +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 969 : backward pass time 10.13 +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 969 : optimizer step time 1.57 +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 969-------------------- +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:43:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683307 37.811642 37.636452 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.603516 133.646484 133.587891 +util 65.000000 70.000000 61.000000 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 970 : avg iteration time 163.35 seconds +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 970 : data loading time 0.00 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 970 : avg forward pass time 73.10 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 970 : backward pass time 8.41 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 970 : optimizer step time 0.41 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 970-------------------- +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 322168.5835 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4472 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 322168.5835iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6756iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4761 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1071 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 629363.0625iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 19.1463iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 11008687.0000iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4472iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.8239 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 283.6906iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2936 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5362.3125iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.4761 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 19.1463 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.0537 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 24521.4043iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:46:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683288 37.811619 37.636406 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.936523 65.101562 64.671875 +util 92.000000 98.000000 84.000000 +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 971 : avg iteration time 47.68 seconds +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 971 : data loading time 0.00 +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 971 : avg forward pass time 12.89 +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 971 : backward pass time 10.15 +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 971 : optimizer step time 1.61 +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 971-------------------- +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0151 iter count: 1.0 +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0151 iter count: 1.0 +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:46:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683741 37.813396 37.636440 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.907227 65.062500 64.671875 +util 95.500000 99.000000 90.000000 +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 972 : avg iteration time 46.02 seconds +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 972 : data loading time 0.00 +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 972 : avg forward pass time 12.07 +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 972 : backward pass time 10.15 +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 972 : optimizer step time 1.56 +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 972-------------------- +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0144 iter count: 1.0 +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0144 iter count: 1.0 +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:47:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683765 37.813400 37.636433 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.755859 65.027344 64.652344 +util 95.250000 100.000000 89.000000 +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 973 : avg iteration time 43.39 seconds +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 973 : data loading time 0.00 +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 973 : avg forward pass time 10.76 +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 973 : backward pass time 10.14 +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 973 : optimizer step time 1.56 +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 973-------------------- +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683797 37.813492 37.636444 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.707031 64.812500 64.671875 +util 96.750000 100.000000 94.000000 +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 974 : avg iteration time 48.71 seconds +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 974 : data loading time 0.00 +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 974 : avg forward pass time 13.42 +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 974 : backward pass time 10.14 +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 974 : optimizer step time 1.58 +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 974-------------------- +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0140 iter count: 1.0 +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:49:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683620 37.811935 37.636780 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.608887 133.652344 133.568359 +util 61.750000 67.000000 58.000000 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 975 : avg iteration time 166.35 seconds +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 975 : data loading time 0.00 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 975 : avg forward pass time 74.55 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 975 : backward pass time 8.45 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 975 : optimizer step time 0.40 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 975-------------------- +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 553295.9443 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4299 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 553295.9443iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.2189iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8183 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3780 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1038309.5000iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.1372 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 18925982.0000iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4299iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.1238 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 28213.4961iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0886 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3559.2302iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.8028 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 271.9239iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.5986 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 72925.2500iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 15:51:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:51:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.24 seconds +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.683690 37.812187 37.636787 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.004883 65.199219 64.730469 +util 95.000000 99.000000 88.000000 +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 976 : avg iteration time 50.67 seconds +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 976 : data loading time 0.00 +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 976 : avg forward pass time 14.30 +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 976 : backward pass time 10.15 +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 976 : optimizer step time 1.57 +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 976-------------------- +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:52:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684128 37.813915 37.636787 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.809570 64.886719 64.671875 +util 92.750000 99.000000 87.000000 +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 977 : avg iteration time 46.08 seconds +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 977 : data loading time 0.00 +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 977 : avg forward pass time 12.11 +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 977 : backward pass time 10.14 +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 977 : optimizer step time 1.58 +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 977-------------------- +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:53:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684138 37.813957 37.636803 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.941406 65.082031 64.769531 +util 92.250000 97.000000 89.000000 +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 978 : avg iteration time 48.83 seconds +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 978 : data loading time 0.00 +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 978 : avg forward pass time 13.48 +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 978 : backward pass time 10.15 +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 978 : optimizer step time 1.56 +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 978-------------------- +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:54:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684140 37.813957 37.636803 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.063477 65.121094 64.929688 +util 96.000000 99.000000 93.000000 +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 979 : avg iteration time 43.46 seconds +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 979 : data loading time 0.00 +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 979 : avg forward pass time 10.79 +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 979 : backward pass time 10.15 +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 979 : optimizer step time 1.58 +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 979-------------------- +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:55:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684364 37.812576 37.637474 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.610352 133.628906 133.570312 +util 67.750000 73.000000 64.000000 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 980 : avg iteration time 168.43 seconds +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 980 : data loading time 0.00 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 980 : avg forward pass time 75.63 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 980 : backward pass time 8.42 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 980 : optimizer step time 0.40 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 980-------------------- +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 355520.5029 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5132 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 355520.5029iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9701iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.2057 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0352 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 564655.4375iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 1.5090 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9534932.0000iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5132iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.2057 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1.5090 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.5287 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3461949.7500iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.7249 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 693657.6250iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.6007 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 181.5347iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:57:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684371 37.812569 37.637474 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.868164 65.160156 64.730469 +util 97.500000 100.000000 95.000000 +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 981 : avg iteration time 47.60 seconds +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 981 : data loading time 0.00 +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 981 : avg forward pass time 12.87 +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 981 : backward pass time 10.14 +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 981 : optimizer step time 1.57 +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 981-------------------- +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0138 iter count: 1.0 +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:58:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684752 37.814079 37.637486 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.873047 65.140625 64.769531 +util 95.500000 100.000000 92.000000 +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 982 : avg iteration time 48.57 seconds +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 982 : data loading time 0.00 +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 982 : avg forward pass time 13.35 +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 982 : backward pass time 10.15 +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 982 : optimizer step time 1.57 +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 982-------------------- +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 15:59:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684754 37.814064 37.637505 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.809570 65.101562 64.671875 +util 94.250000 99.000000 87.000000 +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 983 : avg iteration time 48.70 seconds +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 983 : data loading time 0.00 +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 983 : avg forward pass time 13.41 +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 983 : backward pass time 10.15 +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 983 : optimizer step time 1.58 +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 983-------------------- +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:00:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684819 37.814312 37.637516 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.838867 65.101562 64.710938 +util 96.000000 100.000000 89.000000 +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 984 : avg iteration time 43.35 seconds +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 984 : data loading time 0.00 +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 984 : avg forward pass time 10.75 +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 984 : backward pass time 10.14 +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 984 : optimizer step time 1.56 +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 984-------------------- +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:01:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684452 37.812729 37.637505 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.666992 133.705078 133.646484 +util 67.000000 72.000000 63.000000 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 985 : avg iteration time 165.99 seconds +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 985 : data loading time 0.00 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 985 : avg forward pass time 74.41 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 985 : backward pass time 8.39 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 985 : optimizer step time 0.41 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 985-------------------- +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 161576.5479 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4921 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 161576.5479iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8940iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.8047 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6211 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 295481.3438iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 36.9449iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4165301.0000iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4921iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7840 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 105664.0312iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.3475 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 808.2982iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.2276 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 635.9767iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2383 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 35478.5859iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:03:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684525 37.812984 37.637505 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.711914 64.769531 64.691406 +util 97.000000 100.000000 94.000000 +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 986 : avg iteration time 47.98 seconds +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 986 : data loading time 0.00 +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 986 : avg forward pass time 13.06 +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 986 : backward pass time 10.13 +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 986 : optimizer step time 1.59 +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 986-------------------- +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685128 37.815220 37.637646 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.799805 64.847656 64.714844 +util 94.250000 99.000000 87.000000 +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 987 : avg iteration time 46.02 seconds +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 987 : data loading time 0.00 +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 987 : avg forward pass time 12.05 +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 987 : backward pass time 10.16 +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 987 : optimizer step time 1.59 +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 987-------------------- +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:05:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685122 37.815220 37.637623 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.790039 64.847656 64.671875 +util 96.500000 99.000000 90.000000 +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 988 : avg iteration time 43.43 seconds +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 988 : data loading time 0.00 +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 988 : avg forward pass time 10.78 +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 988 : backward pass time 10.14 +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 988 : optimizer step time 1.58 +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 988-------------------- +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:06:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:06:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685201 37.815472 37.637627 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.780273 64.832031 64.730469 +util 95.000000 99.000000 92.000000 +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 989 : avg iteration time 45.92 seconds +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 989 : data loading time 0.00 +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 989 : avg forward pass time 12.02 +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 989 : backward pass time 10.16 +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 989 : optimizer step time 1.57 +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 989-------------------- +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:06:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684757 37.813637 37.637634 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.608398 133.611328 133.607422 +util 67.000000 72.000000 63.000000 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 990 : avg iteration time 166.05 seconds +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 990 : data loading time 0.00 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 990 : avg forward pass time 74.43 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 990 : backward pass time 8.40 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 990 : optimizer step time 0.40 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 990-------------------- +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 386149.9312 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4752 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 386149.9312iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8599iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0450 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1996 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 780879.8750iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.0844 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 13244812.0000iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4752iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.3403 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 796.8709iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.4781 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3128855.0000iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6986 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 12055.1289iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2364 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 261141.4219iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:09:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684763 37.813644 37.637634 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.838867 64.906250 64.652344 +util 91.750000 97.000000 88.000000 +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 991 : avg iteration time 47.52 seconds +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 991 : data loading time 0.00 +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 991 : avg forward pass time 12.85 +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 991 : backward pass time 10.13 +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 991 : optimizer step time 1.57 +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 991-------------------- +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:10:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685140 37.815121 37.637653 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.921875 65.101562 64.808594 +util 94.750000 99.000000 88.000000 +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 992 : avg iteration time 45.94 seconds +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 992 : data loading time 0.00 +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 992 : avg forward pass time 12.04 +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 992 : backward pass time 10.16 +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 992 : optimizer step time 1.58 +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 992-------------------- +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:11:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685205 37.815384 37.637665 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.014648 65.121094 64.964844 +util 91.000000 96.000000 87.000000 +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 993 : avg iteration time 48.84 seconds +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 993 : data loading time 0.00 +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 993 : avg forward pass time 13.47 +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 993 : backward pass time 10.17 +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 993 : optimizer step time 1.58 +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 993-------------------- +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:12:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685249 37.815407 37.637680 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.004883 65.121094 64.964844 +util 95.750000 100.000000 90.000000 +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 994 : avg iteration time 43.37 seconds +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 994 : data loading time 0.00 +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 994 : avg forward pass time 10.75 +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 994 : backward pass time 10.15 +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 994 : optimizer step time 1.57 +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 994-------------------- +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:12:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684782 37.813511 37.637707 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.593750 133.650391 133.548828 +util 61.500000 67.000000 57.000000 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 995 : avg iteration time 165.85 seconds +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 995 : data loading time 0.00 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 995 : avg forward pass time 74.32 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 995 : backward pass time 8.42 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 995 : optimizer step time 0.40 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 995-------------------- +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 184664.8962 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4519 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 184664.8962iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5065iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2184 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2533 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 539816.4375iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 11.4356iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 14748288.0000iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4519iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.8743 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 42.4638 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2752 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5168.5840iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6848 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 11726.7627iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.7093 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 12315.5410iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:15:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.684791 37.813511 37.637764 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.048828 65.082031 64.984375 +util 95.500000 99.000000 90.000000 +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 996 : avg iteration time 47.75 seconds +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 996 : data loading time 0.00 +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 996 : avg forward pass time 12.93 +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 996 : backward pass time 10.14 +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 996 : optimizer step time 1.58 +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 996-------------------- +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:16:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685052 37.814495 37.637802 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.960938 65.101562 64.769531 +util 96.000000 100.000000 90.000000 +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 997 : avg iteration time 46.20 seconds +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 997 : data loading time 0.00 +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 997 : avg forward pass time 12.16 +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 997 : backward pass time 10.14 +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 997 : optimizer step time 1.58 +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 997-------------------- +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:17:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685232 37.815258 37.637760 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.960938 65.101562 64.769531 +util 96.000000 100.000000 90.000000 +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 998 : avg iteration time 46.00 seconds +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 998 : data loading time 0.00 +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 998 : avg forward pass time 12.05 +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 998 : backward pass time 10.18 +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 998 : optimizer step time 1.55 +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 998-------------------- +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:17:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.685304 37.815514 37.637779 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.980469 65.121094 64.808594 +util 93.250000 100.000000 87.000000 +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 999 : avg iteration time 46.01 seconds +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 999 : data loading time 0.00 +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 999 : avg forward pass time 12.06 +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 999 : backward pass time 10.14 +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 999 : optimizer step time 1.57 +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 999-------------------- +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:18:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 16:18:37 | INFO | fastgen.trainer:run:225 ] Training complete. +[May 5, 2026 - 16:18:37 | INFO | fastgen.trainer:validate:358 ] Validation iteration 1000 +[May 5, 2026 - 16:18:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:18:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.4s +[May 5, 2026 - 16:18:45 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 5, 2026 - 16:18:45 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.012 +[May 5, 2026 - 16:18:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:18:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 5, 2026 - 16:18:50 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.124 +[May 5, 2026 - 16:18:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:18:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 5, 2026 - 16:18:55 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=6.138 +[May 5, 2026 - 16:18:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:18:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.9s +[May 5, 2026 - 16:19:00 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.862 +[May 5, 2026 - 16:19:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:19:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 16:19:05 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.625 +[May 5, 2026 - 16:19:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:19:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 16:19:10 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.078 +[May 5, 2026 - 16:19:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:19:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 16:19:16 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.343 +[May 5, 2026 - 16:19:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:19:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 16:19:21 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.829 +[May 5, 2026 - 16:19:21 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:19:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 16:19:26 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.891 +[May 5, 2026 - 16:19:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 16:19:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 16:19:30 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.500 +[May 5, 2026 - 16:19:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 1000-------------------- +[May 5, 2026 - 16:19:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 16:19:47 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 1000 +[May 5, 2026 - 16:19:47 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.740 (n=10) +[May 5, 2026 - 16:19:47 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 1000 +[May 5, 2026 - 16:19:47 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0001000 +[May 5, 2026 - 16:20:04 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 1000 +[May 5, 2026 - 16:20:04 | INFO | fastgen.trainer:run:232 ] Taking a 10 sec nap and exiting training. +[May 5, 2026 - 16:20:15 | INFO | fastgen.utils.distributed:clean_up:88 ] Distributed clean up. +[May 5, 2026 - 16:20:15 | INFO | __main__::49 ] Training finished. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..02f58552c6a1f697075cd43e96acf406a4f8cc5e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-05T05:22:05.834554Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py", + "-", + "trainer.resume=True", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=1000", + "trainer.save_ckpt_iter=100" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1176050405376" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "z2205v42lvq1qmd2kvnlslezx5dnogze" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8107c044fd058aa5451882e623ab81158223e865 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/reward_sync_c_min":1.2183666229248047,"profiler/backward_pass_time":10.143166679888964,"_step":1000,"_timestamp":1.7779655874790354e+09,"train_media/student/generation":{"_type":"video-file","path":"media/videos/train_media/student/generation_800_6a56be7cdd484957da32.mp4","sha256":"6a56be7cdd484957da3294100dc1510e28d6646918317e2af5b10224a8e8cae7","size":242492},"net/total_params":1421383840,"fake_score/total_params":1421383840,"train/total_loss":0.010711669921875,"model/local_trainable_params":[711111248,710552144,710552144,710552144],"profiler/data_loading_time":0.0007298346608877182,"train/reward_sync_c_r3":4.709308624267578,"val0/sync_c_sample_2":6.138042449951172,"train/reward_sync_c_r0":1.8743257522583008,"val0/sync_c_sample_4":4.624655723571777,"train_media/data/real":{"_type":"video-file","path":"media/videos/train_media/data/real_800_97779d11e00d4fea84a0.mp4","sha256":"97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2","size":194172},"val0/reconstructed":{"captions":false,"_type":"videos","count":10,"videos":[{"_type":"video-file","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028,"path":"media/videos/val0/reconstructed_1000_80aab00fcfa85d5db98e.mp4"},{"path":"media/videos/val0/reconstructed_1000_884b2d197a5511393680.mp4","_type":"video-file","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214","size":170922},{"_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8","size":173236,"path":"media/videos/val0/reconstructed_1000_c0196720a45889bb2198.mp4"},{"sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c","size":249372,"path":"media/videos/val0/reconstructed_1000_294040ac378ceaa34329.mp4","_type":"video-file"},{"path":"media/videos/val0/reconstructed_1000_15ac5b765a401d035db1.mp4","_type":"video-file","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004},{"path":"media/videos/val0/reconstructed_1000_f5463a4ea6f68a39c5d0.mp4","_type":"video-file","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630},{"path":"media/videos/val0/reconstructed_1000_c0f892b5fba78348af85.mp4","_type":"video-file","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986},{"path":"media/videos/val0/reconstructed_1000_a076d17e202ed18ebdf8.mp4","_type":"video-file","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb","size":165516},{"path":"media/videos/val0/reconstructed_1000_6ff046418b5d41477989.mp4","_type":"video-file","sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001},{"_type":"video-file","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490,"path":"media/videos/val0/reconstructed_1000_f7412c9e170ffd299748.mp4"}]},"profiler/optimizer_step_time":1.5737124308943748,"val0/sync_c_sample_0":6.012312889099121,"train/reward_sync_c_max":8.253318786621094,"optimizer/lr_net":2e-06,"train/vsd_loss_weighted":184664.89624023438,"model/total_params":17150480992,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"fake_score/trainable_params":1421383840,"optimizer/non_finite_grads_count (model_key net)":0,"val0/sync_c_sample_8":5.891082286834717,"train/reward_sync_c_r1":4.275177001953125,"val0/sync_c_sample_1":6.123776435852051,"train/vsd_loss_unweighted":0.451904296875,"train/reward_weight_r1":5168.583984375,"train/reward_weight_r2":11726.7626953125,"val0/sync_c_mean":5.740166330337525,"optimizer/grad_norm (model_key net)":0,"train/reward_weight_r0":42.46377944946289,"optimizer/lr_fake_score":2e-06,"_wandb":{"runtime":79191},"train/gan_loss_gen":0,"train/reward_weight_mean":539816.4375,"model/local_total_params":[4299192568,4297394424,4297394424,4297394424],"val0/sync_c_sample_3":5.86152458190918,"profiler/avg_forward_pass_time":12.062476909719408,"val0/sync_c_sample_7":5.828784942626953,"model/trainable_params":2842767680,"optimizer/iteration":1000,"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"val0/sync_c_sample_6":6.34311580657959,"train/fake_score_loss":0.010711669921875,"train/vsd_loss":0.451904296875,"val0/total_loss":0,"train/reward_sync_c_r2":4.684814453125,"net/local_total_params":[355555624,355276072,355276072,355276072],"val0/generated":{"videos":[{"sha256":"7f1688b26c115ba620a57db2534d1d09c5dc04798c86d4b94a3379504649a26a","size":218000,"path":"media/videos/val0/generated_1000_7f1688b26c115ba620a5.mp4","caption":"sync_c=6.012","_type":"video-file"},{"_type":"video-file","sha256":"776855bea09e3ee28d4faca7baa99c6096a321e6de248d89d5055f4f0b34b34e","size":222921,"path":"media/videos/val0/generated_1000_776855bea09e3ee28d4f.mp4","caption":"sync_c=6.124"},{"size":208735,"path":"media/videos/val0/generated_1000_1d5d847d7164bb28616f.mp4","caption":"sync_c=6.138","_type":"video-file","sha256":"1d5d847d7164bb28616fe152af9db791eb56fe5fb0ba93be150a376f5f7e9d08"},{"_type":"video-file","sha256":"7105cc2283e6acf8678a347bea69119f41eaef4741e3561e0b0967674de01b45","size":299641,"path":"media/videos/val0/generated_1000_7105cc2283e6acf8678a.mp4","caption":"sync_c=5.862"},{"sha256":"7a8d3fc9242239c85404ca99373c3cfbb5c154df530b2dd222c65e8acfc75a3a","size":261519,"path":"media/videos/val0/generated_1000_7a8d3fc9242239c85404.mp4","caption":"sync_c=4.625","_type":"video-file"},{"size":185990,"path":"media/videos/val0/generated_1000_042b02b2ceeb648cf655.mp4","caption":"sync_c=7.078","_type":"video-file","sha256":"042b02b2ceeb648cf65516bc0efc2bf1fe36e19364265bff8c4de13fdac76ab9"},{"sha256":"e55b0b718380e13aab7d29c90c6c66173eddf8b51a2d1ffc2c61e629d0212dc7","size":304501,"path":"media/videos/val0/generated_1000_e55b0b718380e13aab7d.mp4","caption":"sync_c=6.343","_type":"video-file"},{"sha256":"4b40b35b2d64012475c8b8e1a88fbf5e8a6c8ff6b45f0e3d3867c7f945bc7710","size":207550,"path":"media/videos/val0/generated_1000_4b40b35b2d64012475c8.mp4","caption":"sync_c=5.829","_type":"video-file"},{"caption":"sync_c=5.891","_type":"video-file","sha256":"cdda7c097236ecac61ec1ffd5061031d0134fad2bfe519fcc070ed7db02c2972","size":241147,"path":"media/videos/val0/generated_1000_cdda7c097236ecac61ec.mp4"},{"path":"media/videos/val0/generated_1000_1bd4125209338a80be7e.mp4","caption":"sync_c=3.500","_type":"video-file","sha256":"1bd4125209338a80be7e813303808aab3f1488b772b74d7029376a714b9687e9","size":248914}],"captions":["sync_c=6.012","sync_c=6.124","sync_c=6.138","sync_c=5.862","sync_c=4.625","sync_c=7.078","sync_c=6.343","sync_c=5.829","sync_c=5.891","sync_c=3.500"],"_type":"videos","count":10},"train/reward_weight_r3":12315.541015625,"val0/sync_c_sample_9":3.500286102294922,"train/gan_loss_disc":0,"_runtime":79191.096490012,"net/trainable_params":1421383840,"train/reward_sync_c_mean":4.506463050842285,"train/reward_weight_max":1.4748288e+07,"train/reward_weight_min":11.435623168945312,"profiler/avg_iteration_time":46.014070987701416,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"val0/sync_c_sample_5":7.078082084655762} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..835de7f9ebd0cbc554d87c55d9e9a58ac3405f9a --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-05T14:22:06.19500055+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpfgab88bg/port-264993.txt","pid":264993,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-05T14:22:06.195983614+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":264993} +{"time":"2026-05-05T14:22:06.1954799+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-264993-284038-2036868350/socket","Net":"unix"}} +{"time":"2026-05-05T14:22:06.31052484+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-05T14:22:06.329958803+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T14:22:06.836983028+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-05-05T14:22:12.927777205+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6mvm4oh57m57"} +{"time":"2026-05-05T16:20:15.57069052+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T16:20:15.570758875+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T16:20:15.57074988+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T16:20:15.570839734+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-264993-284038-2036868350/socket","Net":"unix"}} +{"time":"2026-05-05T16:20:15.570895558+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T16:20:16.906214523+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-05T16:20:16.906231908+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-05T16:20:16.906239807+09:00","level":"INFO","msg":"server is closed"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c2f4ccd8a3d708e5e5b5672f2085fb3e1a41a710 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T14:22:06.33051782+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T14:22:06.836248611+09:00","level":"INFO","msg":"stream: created new stream","id":"lvwg9pfc"} +{"time":"2026-05-05T14:22:06.83692293+09:00","level":"INFO","msg":"handler: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T14:22:06.836976277+09:00","level":"INFO","msg":"stream: started","id":"lvwg9pfc"} +{"time":"2026-05-05T14:22:06.836995755+09:00","level":"INFO","msg":"sender: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T14:22:06.837022091+09:00","level":"INFO","msg":"writer: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T16:20:15.570749087+09:00","level":"INFO","msg":"stream: closing","id":"lvwg9pfc"} +{"time":"2026-05-05T16:20:16.516503136+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T16:20:16.905948798+09:00","level":"INFO","msg":"handler: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T16:20:16.906020747+09:00","level":"INFO","msg":"sender: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-05T16:20:16.906038102+09:00","level":"INFO","msg":"stream: closed","id":"lvwg9pfc"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a81103634c834c7b3eb6918f322689edbdea04de --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug.log @@ -0,0 +1,22 @@ +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_setup.py:_flush():81] Configure stats pid to 264993 +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug.log +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/logs/debug-internal.log +2026-05-05 14:22:05,835 INFO MainThread:264993 [wandb_init.py:init():844] calling init triggers +2026-05-05 14:22:05,836 INFO MainThread:264993 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '1000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 14:22:05,836 INFO MainThread:264993 [wandb_init.py:init():892] starting backend +2026-05-05 14:22:06,310 INFO MainThread:264993 [wandb_init.py:init():895] sending inform_init request +2026-05-05 14:22:06,325 INFO MainThread:264993 [wandb_init.py:init():903] backend started and connected +2026-05-05 14:22:06,329 INFO MainThread:264993 [wandb_init.py:init():973] updated telemetry +2026-05-05 14:22:06,347 INFO MainThread:264993 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 14:22:07,470 INFO MainThread:264993 [wandb_init.py:init():1037] run resumed +2026-05-05 14:22:07,473 INFO MainThread:264993 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 14:22:07,919 INFO MainThread:264993 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 14:22:07,919 INFO MainThread:264993 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 14:22:07,919 INFO MainThread:264993 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 14:22:07,919 INFO MainThread:264993 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 14:22:07,930 INFO MainThread:264993 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 16:20:15,570 INFO wandb-AsyncioManager-main:264993 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 16:20:15,570 INFO wandb-AsyncioManager-main:264993 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/run-lvwg9pfc.wandb b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/run-lvwg9pfc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5d49b4350f9de40cae4a7d8137503234cc31c313 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260505_142205-lvwg9pfc/run-lvwg9pfc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:def64c17107d1fb158d78e4595a0a91b176a9470b0e223bdb300a8c73a12aa5f +size 1653150 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb_id.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..9b47f34743d410b917507f7832718802f87729cb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_CONTINUE_600_TO_900/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb_id.txt @@ -0,0 +1 @@ +lvwg9pfc diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..9b12caee7c26f7909e7570111de7d045076a8f0b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e1b45a71b99a9d79398ef95728951fbdbbd83a0d3ed6879e1812a2fc4416472 +size 635541 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9323da170f9f68a20b6cc3e809539fe7a59b6ea4 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0336ef86b2855aea94cfbc1cce755f9e0f804c4e61dff108a1df016fef5cdd34 +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..5c5575465d14a1852e241a364a4eb64ffdac9e09 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dbf0c1bbf1f5d68c85d92d2e61c260ee71e57a21dd132a04c519388b141f9a8 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2e36d09ab3d4acc11131a9b49819d78c8f6e6b67 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916b2a5271df34e5461288ac4e08e17b7d2e8887e51c1747c4e374e2148f7f53 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..95b5df042b63a14ea0a61243ee77edbf84e44cea --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c02b49a83b410452f04681ea815cde5e2ff86dcd31be993ed8b7f0cbffea7a1f +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c4e1e634f09607df0ebed8f945c77fd15973a5b3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31d51db3f56a302f912072636dda8f7d17c5d37c346fe2be929f4c9155dbcda4 +size 1722905 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f15f47e61aa74245e22d3601527ff497bb48718c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e231f513999d68a0e20bc37507bff92d21c8cab5ee9e957e4e852083fa6723 +size 2847149355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9a935650d14507c36bc66cb00510e376a1238430 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16c5f469fb0f8896e7b8e129182051390611f299b1f9f1d87232bbb57c45d0b3 +size 2845267132 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a8c6292f5a2b0f23ae7c246b1f60c066c2b93a2b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bdc4dda755056ed6680e8bbe38d8432bc35c29d5e47743b08b571b459ded02e +size 2845352958 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c28800c3898168a7f1d2ca63ea734fb3004db9d7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c22c1b04e5cb5f1f1ad15dd288101f7f7aee48d9885188da6d78372076e95555 +size 2845316241 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1d23fbf4920c6264d7cc54b5134b03b03add6043 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2846790d16d780c5a3a8ef6217367037c9fd8632f4170b1d8ed946e2ce59f1d9 +size 635534 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4782a9c165411f1e81af7f8202c2df7392c1bfc1 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c323cecc51002060b8fe7173d46391685c7b5ef48fc9c6d1f9e1c65424226aec +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..48601019f985328cbcf496ea282c689bd329afd0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3f776f6cfa7b78a82e890baf113262b0e0d154578e38a73aad63e2601a540bd +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..91d8fae4e978425f7527062733211821d7e47809 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3a087bd9d2c6176e160335749488aff328be6454231b4c919edd3361b0b3368 +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..244e8d007708ce1ba50b9a9e973818f39bc2fc61 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd2d34229935df2d32634decbdbf35dfd84115dd75a3df87a2912c4d2ea29168 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..af39f975341150f46a3449d81af2f3d200e12b47 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44acee2cfca49ad32dbf926bac03b2c7506750bb5be553a6f082436d879337a3 +size 1722748 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a7ff6e6e7419c4954e877c4bf70117c5c356d2bf --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f92de15d765b20c95a85e13755457394fd6443f8fae158d526d70e26a97c553a +size 2847149355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..280e976c777bdd753a29c5ce5ec1e650d04b79ef --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24ead14bc101bff52c2eb50a46bac684529d2016b6dc226bbf181d8f289d4a49 +size 2845267132 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3e3555e4c77bf51a628614620f5c4160d272c9a8 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5b428341f0a2a3c46f2c92ccfb8c7cd9aaa1eb89437a0665ac3f7740d3ab979 +size 2845352958 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fbb165baa80ca3c236dfe8f9829c97824360dcb9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d414c2de330d4c4601d18ada8c9c843a7fa668fbce439f8a2e9009d6006ef39 +size 2845316241 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth new file mode 100644 index 0000000000000000000000000000000000000000..20e4f38a6d8cc3aa8546b004ca664ad199f33358 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d438cc8922e447919e87494a29301517f583bd61e7ff6182b950306238a2117d +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d90bd8051d0224457de81e2651c2d48aa0ff9b4e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'False' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '0' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.769' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '800' + offload_module_in_decoding: 'False' + resume: 'True' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..d5e5634df9b8c67d5e17cccf6769e509b27b3bf9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt @@ -0,0 +1 @@ +lw2bm1l6 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth new file mode 100644 index 0000000000000000000000000000000000000000..20e4f38a6d8cc3aa8546b004ca664ad199f33358 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d438cc8922e447919e87494a29301517f583bd61e7ff6182b950306238a2117d +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5f7028a189c49087921ceb9561e07c2961984101 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a6c10ba3b2051edfec00c589277db04be8a3a43ce3f223b69203e7dd6d2f3a5 +size 635558 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..47ffecb99fb3e5e8c3e4a454e8c50bcc1fc7a79e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f47c7739df66bf704155f7c27066f0827bb0a2562b112e41fc73432456e1aa +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1cdef7e44e72844bc92b2a92f0ba8750c83e0c19 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdfbc4cf503b32e0d649c3546715983920d172b1e23b493b7541b16a1d277d3b +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e7c3058121ae86f4ec2eeb12a587ece0ba3f2a22 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71da34b92d7e1a5c670ef3f9d82fbb51070a22296f7e355657ea6c9d3e273ac +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1b49b003a9c461244db77add42a90c95f5b3e6d7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9081ad46bd45f6cc6d208dfcbea61b81e1e93175580a984de6a5d5dfcc1d0aa +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5553f3a8798d0c726585960409351575d75e270b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6cb27a66c9fec1a0ec3facc2a58a08d246f74676a273e135a6349c2d239f5be +size 1722922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6db9b054ef6f29ee5c802bd25d68b4683b979032 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c94af1ee33aca06fe8425a4d2bff78afb4bf2cdd6195d859ca8071f2ed948f3 +size 2847149355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8e63e5578a264e3ef7a8f99a9d75986c0a28d167 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2088eb856801e757e559a32204fa21f8e1969960d378fef287ca98d95655996e +size 2845267132 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2468b288e3c0a07f46de5dd6a3f9a712a8799cce --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f489e8336f3323f864dc050a62327d48c78fdb25ac5fef3d2ff3ead4e58c08eb +size 2845352958 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b3603e369e1a2e3fcbf3fd2ad4bb3e650f0f3517 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03dc03bdd75bbcb275c494149832dd34beb8f9ab41ad56c70e5da4c2e269d2ca +size 2845316241 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..28398d844e73a50d0be0b87c81c01c0cd2fe985f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5150208a12ac95acd164f6f98a9c829bea49d2f7fb5237e3e01d71175d0f618b +size 635551 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..98f302d0acd79a94803927da9b22bdf380d2b52c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:415aa5b74b65854a38b399a8a2baa3a2a9ad45396e0559accc81737d86ef704e +size 1423573889 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4d11eef99f658f548fbbbf2eda66ee21d3002cf7 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1565f074d3c1c60249b644b40676669b298099590c5caf43ee15eece9d4e353 +size 1422406794 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7e283b31c806aaac7faee61065d76b62e87e6710 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a9550279328c9d69d6586e88a3f6979f204dc9172b068c19b078576d4acd27d +size 1422431002 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ef05bd2dc61dbabdde98f142c6e8497c4a971543 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33f592e44df988a1c11937f1c4efa58d021c92046a1a01fd9a37de532d1b6b92 +size 1422429489 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/.metadata b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..3c62dd8a18efca00c464422096ad1c8cf51ac29c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbabc686989f25f5ca32bf90d77b1215c578516437b885bb2bec72caf3a747a1 +size 1722765 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__0_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e038ac5d903b0b8a6bb263a5e8f97b4908cf5323 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0fd68877b90e3a539be0025dc78d07f5ef2581f5005ff2c3bda8acad840e3da +size 2847149355 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__1_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..05daf12e56c4bec281a32f369e3357f540d7aea6 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68ae27d163eb1a9d471ea6973b5f94d8261efbe69e93e5c8d2df19ac2aa2ddb5 +size 2845267132 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__2_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ada6feda9e4b775a4a7920d361231f1606f6f88f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2768abaf0a38d8eab0cdc2cd4c5259e49d9887f9623c91030a440462e3d55611 +size 2845352958 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__3_0.distcp b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..01c54a0dbce6e4f0169dce64a03e103b4593034b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eede84a9aa1ce455a1c2d09df4223874d582780836160de3212bf953a6aaf1e3 +size 2845316241 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.pth b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.pth new file mode 100644 index 0000000000000000000000000000000000000000..86ac3c3f3f9acfc8ddd7ab9c162e0eb82d0c2a4e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4489755a2dfc00f4441b0a2ee607f08a83fe31ff32f53f07ef4111b972079cfb +size 1901 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..312537c2dad6b77b1339aa1b57a06e4721c9063b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'False' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '0' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.769' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '800' + offload_module_in_decoding: 'False' + resume: 'True' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fb2cd06cc004ee799e3e0a5fba0e523ae3e25759 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T16:57:01.103690934+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T16:57:01.512285583+09:00","level":"INFO","msg":"stream: created new stream","id":"lw2bm1l6"} +{"time":"2026-05-05T16:57:01.513131599+09:00","level":"INFO","msg":"handler: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T16:57:01.513185968+09:00","level":"INFO","msg":"stream: started","id":"lw2bm1l6"} +{"time":"2026-05-05T16:57:01.513204736+09:00","level":"INFO","msg":"sender: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T16:57:01.513240232+09:00","level":"INFO","msg":"writer: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T18:54:12.660302666+09:00","level":"INFO","msg":"stream: closing","id":"lw2bm1l6"} +{"time":"2026-05-05T18:54:13.880823166+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T18:54:16.418725916+09:00","level":"INFO","msg":"handler: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T18:54:16.419293593+09:00","level":"INFO","msg":"sender: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T18:54:16.419315458+09:00","level":"INFO","msg":"stream: closed","id":"lw2bm1l6"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a56b6b2b4a4248ca1266c2f1b9f13e3a38af5f87 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/debug.log @@ -0,0 +1,22 @@ +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_setup.py:_flush():81] Configure stats pid to 784820 +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug.log +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug-internal.log +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_init.py:init():844] calling init triggers +2026-05-05 16:57:00,625 INFO MainThread:784820 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '800', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 16:57:00,625 INFO MainThread:784820 [wandb_init.py:init():892] starting backend +2026-05-05 16:57:01,082 INFO MainThread:784820 [wandb_init.py:init():895] sending inform_init request +2026-05-05 16:57:01,097 INFO MainThread:784820 [wandb_init.py:init():903] backend started and connected +2026-05-05 16:57:01,100 INFO MainThread:784820 [wandb_init.py:init():973] updated telemetry +2026-05-05 16:57:01,121 INFO MainThread:784820 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 16:57:02,180 INFO MainThread:784820 [wandb_init.py:init():1037] run resumed +2026-05-05 16:57:02,183 INFO MainThread:784820 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 16:57:02,436 INFO MainThread:784820 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 16:57:02,436 INFO MainThread:784820 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 16:57:02,436 INFO MainThread:784820 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 16:57:02,436 INFO MainThread:784820 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 16:57:02,441 INFO MainThread:784820 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 18:54:12,660 INFO wandb-AsyncioManager-main:784820 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 18:54:12,662 INFO wandb-AsyncioManager-main:784820 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2821a48d59cd378b96f3f06a2c159ee30fc9a2de --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/config.yaml @@ -0,0 +1,391 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + ijbznehtvc6ciol7xuxzz1ktw5k81eu8: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769_no_reward.py + - '-' + - trainer.resume=True + - log_config.group=omniavatar_sf_full_ft + - log_config.name=sf_full_ft_t769_no_reward + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=800 + - trainer.save_ckpt_iter=100 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1176060735488" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward + startedAt: "2026-05-05T07:57:00.623568Z" + writerId: ijbznehtvc6ciol7xuxzz1ktw5k81eu8 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769_no_reward + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "False" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "0" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "800" + offload_module_in_decoding: "False" + resume: "True" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_18cdb0fb0acea943ae1b.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_18cdb0fb0acea943ae1b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbbcbd35818726d0ef867e1224468142125577c6 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_18cdb0fb0acea943ae1b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18cdb0fb0acea943ae1be8bc98f8554d890bed7d4c5686dd660c3e3ef3567026 +size 243732 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_56f78017d0746eb5d2dc.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_56f78017d0746eb5d2dc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1644f5e8e3485ad955ed585115930bb8bdc8c355 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_56f78017d0746eb5d2dc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56f78017d0746eb5d2dc188735b0cf6938615b98c2a11286b10826e83c91fb35 +size 216854 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_59adee4c2c1233294521.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_59adee4c2c1233294521.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a7909e3a2365daffa266235cd1f212af31407fb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_59adee4c2c1233294521.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59adee4c2c12332945212a9f37785447fde66ae3136b437df8ca1975a263c091 +size 210861 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_6853c5e28978367eb479.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_6853c5e28978367eb479.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..601c1c4d5c628944d870528983d83ea792176afb --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_6853c5e28978367eb479.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6853c5e28978367eb479667a6d60b6fa991de2e9ee5c993d42f2654567917be2 +size 219627 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_75bb737f23f37c356994.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_75bb737f23f37c356994.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5752cb0657f054aa8dcd56235f0b27357fad8e08 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_75bb737f23f37c356994.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75bb737f23f37c356994ec54519c0b7eb03e9879031d9093cfdab75250db3c2c +size 301276 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_8180e794591d55c93416.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_8180e794591d55c93416.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..161da837c7cb4c10ec6daa743ff239dcdcc6056d --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_8180e794591d55c93416.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8180e794591d55c93416e298518e6c3b53acab661fd0b2a3ec3fdcc61059b98b +size 200578 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_85b5b43e4c471767517f.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_85b5b43e4c471767517f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ccd5f62adc9eb0b5a9604f9ed6607173fad8f879 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_85b5b43e4c471767517f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85b5b43e4c471767517fa55c958b4d0bbcee7d66b96b35efd6cef44a3a11d3d1 +size 246015 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_b2a3a97608a2bdd39582.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_b2a3a97608a2bdd39582.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9031b8ac3d3486516cf79bded5579e5ecaf03c40 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_b2a3a97608a2bdd39582.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2a3a97608a2bdd39582fba0c457a16625d74b22ddb8a771b9fc0c2ca0225a1b +size 302581 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_b4cb8b6f832fd8a17929.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_b4cb8b6f832fd8a17929.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fbd8ba59fafa9603339f86bfb8a9becb69810164 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_b4cb8b6f832fd8a17929.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4cb8b6f832fd8a179298dc8b3f479408dd7f3ddd21aadad7cf2c64414218e79 +size 260341 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_dfe41c776e29223e9b8c.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_dfe41c776e29223e9b8c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de323bce97c89890c93c5553737f35283efc66f0 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/generated_800_dfe41c776e29223e9b8c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfe41c776e29223e9b8cef0d4076ba721edf4dd6f0a6b2566f3b0769f9070ab2 +size 192865 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/output.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0e579dfa712c045e15c2a9c84a12b1b6c2524e9f --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/output.log @@ -0,0 +1,1828 @@ +[May 5, 2026 - 16:57:02 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 5, 2026 - 16:57:02 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 5, 2026 - 16:57:02 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 5, 2026 - 16:57:02 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 5, 2026 - 16:57:02 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 5, 2026 - 16:57:02 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 5, 2026 - 16:57:02 | INFO | fastgen.trainer:run:77 ] Starting training +[May 5, 2026 - 16:57:02 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 5, 2026 - 16:57:02 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[May 5, 2026 - 16:57:31 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[May 5, 2026 - 16:57:43 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[May 5, 2026 - 16:57:43 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[May 5, 2026 - 16:57:43 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[May 5, 2026 - 16:57:43 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 5, 2026 - 16:58:26 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 5, 2026 - 16:58:26 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 5, 2026 - 16:58:26 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 5, 2026 - 16:58:26 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 5, 2026 - 16:58:26 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[May 5, 2026 - 16:58:26 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 5, 2026 - 16:58:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 16:58:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[May 5, 2026 - 16:58:29 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[May 5, 2026 - 16:58:29 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 16:58:30 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 16:58:30 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 1.9s +[May 5, 2026 - 16:58:30 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 5, 2026 - 16:58:30 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 16:58:42 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 16:58:43 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 12.2s +[May 5, 2026 - 16:58:43 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 5, 2026 - 16:58:43 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 5, 2026 - 16:58:43 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 5, 2026 - 16:58:43 | INFO | fastgen.trainer:run:137 ] Loading checkpoints for resuming .. +[May 5, 2026 - 16:58:43 | INFO | fastgen.utils.checkpointer:load:436 ] Loading FSDP model from prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000700 +[May 5, 2026 - 16:58:43 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key net... +[May 5, 2026 - 16:58:51 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key fake_score... +[May 5, 2026 - 16:59:00 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key net... +[May 5, 2026 - 16:59:00 | SUCCESS | fastgen.utils.checkpointer:load:544 ] Successfully loaded optimizer state for net +[May 5, 2026 - 16:59:00 | INFO | fastgen.utils.checkpointer:load:512 ] Loading the FSDP optimizer dict for key fake_score... +[May 5, 2026 - 16:59:00 | SUCCESS | fastgen.utils.checkpointer:load:544 ] Successfully loaded optimizer state for fake_score +[May 5, 2026 - 16:59:00 | INFO | fastgen.utils.checkpointer:load:565 ] Loading the scheduler_dict... +[May 5, 2026 - 16:59:00 | INFO | fastgen.utils.checkpointer:load:573 ] Loading the gradient scaler... +[May 5, 2026 - 16:59:00 | WARNING | fastgen.utils.checkpointer:load:578 ] Gradient scaler state is empty (likely saved from disabled scaler), skipping load. +[May 5, 2026 - 16:59:00 | INFO | fastgen.utils.checkpointer:load:581 ] Loading the callbacks... +[May 5, 2026 - 16:59:00 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 700. +[May 5, 2026 - 16:59:00 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17136.85 M total params (logical). +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4285.56 M total params LOCAL on rank 0. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 1. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 2. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4283.76 M total params LOCAL on rank 3. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[May 5, 2026 - 16:59:06 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[May 5, 2026 - 16:59:06 | INFO | fastgen.trainer:run:174 ] iter_start: 700 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.466653 37.580460 37.387745 +peak_gpu_mem_gb 58.334157 58.335719 58.333636 +peak_gpu_mem_reserved_gb 60.547363 60.548828 60.546875 +util 75.500000 81.000000 71.000000 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 701 : data loading time 1.23 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 701 : avg forward pass time 10.17 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 701 : backward pass time 10.12 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 701 : optimizer step time 1.76 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 701-------------------- +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 16:59:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.641347 37.755711 37.562321 +peak_gpu_mem_gb 60.982488 60.984831 60.981707 +peak_gpu_mem_reserved_gb 62.289551 62.291016 62.289062 +util 94.750000 100.000000 88.000000 +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 702 : avg iteration time 43.33 seconds +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 702 : data loading time 0.00 +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 702 : avg forward pass time 10.70 +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 702 : backward pass time 10.18 +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 702 : optimizer step time 1.58 +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 702-------------------- +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:00:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.641602 37.756317 37.562542 +peak_gpu_mem_gb 60.982488 60.984831 60.981707 +peak_gpu_mem_reserved_gb 62.328613 62.330078 62.328125 +util 95.500000 99.000000 92.000000 +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 703 : avg iteration time 46.11 seconds +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 703 : data loading time 0.00 +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 703 : avg forward pass time 12.11 +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 703 : backward pass time 10.15 +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 703 : optimizer step time 1.58 +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 703-------------------- +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:01:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.642032 37.756496 37.563576 +peak_gpu_mem_gb 60.982488 60.984831 60.981707 +peak_gpu_mem_reserved_gb 62.289551 62.291016 62.289062 +util 94.750000 100.000000 88.000000 +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 704 : avg iteration time 45.89 seconds +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 704 : data loading time 0.00 +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 704 : avg forward pass time 12.00 +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 704 : backward pass time 10.16 +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 704 : optimizer step time 1.56 +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 704-------------------- +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:02:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.657609 37.769821 37.579571 +peak_gpu_mem_gb 96.120852 96.123195 96.120071 +peak_gpu_mem_reserved_gb 103.186523 103.189453 103.185547 +util 57.500000 64.000000 55.000000 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 705 : avg iteration time 162.12 seconds +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 705 : data loading time 0.00 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 705 : avg forward pass time 72.13 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 705 : backward pass time 8.51 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 705 : optimizer step time 0.43 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 705-------------------- +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4990 iter count: 1.0 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4990 iter count: 1.0 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4990 iter count: 1.0 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4990iter count: 1.0 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:04:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.657838 37.770199 37.579853 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.543945 64.546875 64.542969 +util 91.250000 97.000000 87.000000 +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 706 : avg iteration time 47.14 seconds +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 706 : data loading time 0.00 +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 706 : avg forward pass time 12.64 +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 706 : backward pass time 10.14 +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 706 : optimizer step time 1.59 +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 706-------------------- +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:05:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.658203 37.770203 37.579857 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.954102 64.957031 64.953125 +util 95.750000 99.000000 93.000000 +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 707 : avg iteration time 43.46 seconds +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 707 : data loading time 0.00 +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 707 : avg forward pass time 10.80 +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 707 : backward pass time 10.14 +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 707 : optimizer step time 1.57 +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 707-------------------- +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:06:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.658793 37.772045 37.580013 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.504883 64.507812 64.503906 +util 96.500000 99.000000 90.000000 +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 708 : avg iteration time 43.47 seconds +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 708 : data loading time 0.00 +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 708 : avg forward pass time 10.80 +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 708 : backward pass time 10.15 +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 708 : optimizer step time 1.57 +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 708-------------------- +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:07:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.658879 37.772083 37.580029 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.504883 64.507812 64.503906 +util 93.250000 99.000000 88.000000 +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 709 : avg iteration time 43.46 seconds +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 709 : data loading time 0.00 +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 709 : avg forward pass time 10.80 +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 709 : backward pass time 10.15 +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 709 : optimizer step time 1.56 +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 709-------------------- +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:07:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.659856 37.772091 37.581116 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.161621 105.166016 105.160156 +util 72.750000 95.000000 63.000000 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 710 : avg iteration time 175.08 seconds +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 710 : data loading time 0.00 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 710 : avg forward pass time 77.67 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 710 : backward pass time 10.17 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 710 : optimizer step time 0.39 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 710-------------------- +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4761 iter count: 1.0 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4761 iter count: 1.0 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4761 iter count: 1.0 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4761iter count: 1.0 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:10:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.660083 37.772778 37.581230 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.885742 64.894531 64.859375 +util 95.000000 100.000000 91.000000 +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 711 : avg iteration time 52.98 seconds +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 711 : data loading time 0.00 +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 711 : avg forward pass time 14.25 +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 711 : backward pass time 10.86 +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 711 : optimizer step time 1.80 +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 711-------------------- +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0147 iter count: 1.0 +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0147 iter count: 1.0 +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:11:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.660234 37.773117 37.581352 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 66.038086 66.457031 64.781250 +util 98.750000 100.000000 97.000000 +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 712 : avg iteration time 47.68 seconds +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 712 : data loading time 0.00 +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 712 : avg forward pass time 12.00 +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 712 : backward pass time 10.79 +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 712 : optimizer step time 1.77 +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 712-------------------- +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:12:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.660671 37.774387 37.581684 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.861328 64.933594 64.644531 +util 93.500000 100.000000 89.000000 +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 713 : avg iteration time 54.90 seconds +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 713 : data loading time 0.00 +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 713 : avg forward pass time 14.73 +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 713 : backward pass time 11.83 +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 713 : optimizer step time 1.79 +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 713-------------------- +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:13:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.660824 37.774834 37.581753 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871094 64.914062 64.742188 +util 94.000000 99.000000 90.000000 +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 714 : avg iteration time 51.01 seconds +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 714 : data loading time 0.00 +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 714 : avg forward pass time 12.87 +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 714 : backward pass time 11.65 +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 714 : optimizer step time 1.78 +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 714-------------------- +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:14:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.660927 37.773647 37.582138 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.205566 105.224609 105.199219 +util 66.750000 75.000000 61.000000 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 715 : avg iteration time 179.21 seconds +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 715 : data loading time 0.00 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 715 : avg forward pass time 79.80 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 715 : backward pass time 9.40 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 715 : optimizer step time 0.39 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 715-------------------- +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5625 iter count: 1.0 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5625 iter count: 1.0 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5625 iter count: 1.0 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5625iter count: 1.0 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:17:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.661038 37.773655 37.582256 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.626953 64.820312 64.562500 +util 95.500000 100.000000 89.000000 +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 716 : avg iteration time 50.12 seconds +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 716 : data loading time 0.00 +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 716 : avg forward pass time 13.79 +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 716 : backward pass time 10.14 +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 716 : optimizer step time 1.57 +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 716-------------------- +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:17:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.661413 37.774887 37.582378 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.719727 64.722656 64.718750 +util 97.250000 100.000000 94.000000 +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 717 : avg iteration time 47.51 seconds +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 717 : data loading time 0.00 +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 717 : avg forward pass time 12.12 +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 717 : backward pass time 11.38 +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 717 : optimizer step time 1.74 +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 717-------------------- +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:18:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.661836 37.775013 37.582592 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.548828 64.683594 64.503906 +util 87.250000 93.000000 79.000000 +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 718 : avg iteration time 49.27 seconds +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 718 : data loading time 0.00 +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 718 : avg forward pass time 13.57 +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 718 : backward pass time 10.14 +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 718 : optimizer step time 1.70 +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 718-------------------- +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:19:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.661852 37.774998 37.582649 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.641602 64.644531 64.640625 +util 94.500000 100.000000 87.000000 +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 719 : avg iteration time 51.80 seconds +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 719 : data loading time 0.00 +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 719 : avg forward pass time 14.97 +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 719 : backward pass time 10.15 +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 719 : optimizer step time 1.58 +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 719-------------------- +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:20:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662033 37.774879 37.583054 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.308105 105.810547 105.140625 +util 65.000000 71.000000 58.000000 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 720 : avg iteration time 160.41 seconds +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 720 : data loading time 0.00 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 720 : avg forward pass time 71.61 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 720 : backward pass time 8.40 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 720 : optimizer step time 0.39 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 720-------------------- +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5596 iter count: 1.0 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5596 iter count: 1.0 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5596 iter count: 1.0 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5596iter count: 1.0 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:23:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662142 37.775063 37.583096 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.607422 64.621094 64.566406 +util 94.000000 99.000000 90.000000 +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 721 : avg iteration time 47.07 seconds +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 721 : data loading time 0.00 +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 721 : avg forward pass time 12.61 +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 721 : backward pass time 10.15 +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 721 : optimizer step time 1.57 +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 721-------------------- +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:23:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662449 37.776146 37.583149 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.705078 65.015625 64.601562 +util 94.250000 99.000000 87.000000 +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 722 : avg iteration time 48.72 seconds +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 722 : data loading time 0.00 +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 722 : avg forward pass time 13.44 +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 722 : backward pass time 10.13 +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 722 : optimizer step time 1.56 +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 722-------------------- +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:24:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662688 37.777012 37.583149 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.670898 64.996094 64.562500 +util 97.000000 99.000000 94.000000 +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 723 : avg iteration time 45.94 seconds +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 723 : data loading time 0.00 +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 723 : avg forward pass time 12.05 +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 723 : backward pass time 10.13 +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 723 : optimizer step time 1.56 +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 723-------------------- +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662838 37.777409 37.583149 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.705078 65.015625 64.601562 +util 94.250000 99.000000 87.000000 +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 724 : avg iteration time 43.41 seconds +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 724 : data loading time 0.00 +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 724 : avg forward pass time 10.77 +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 724 : backward pass time 10.14 +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 724 : optimizer step time 1.57 +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 724-------------------- +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:26:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662458 37.775665 37.583157 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.332520 105.355469 105.263672 +util 65.500000 71.000000 59.000000 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 725 : avg iteration time 160.43 seconds +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 725 : data loading time 0.00 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 725 : avg forward pass time 71.59 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 725 : backward pass time 8.41 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 725 : optimizer step time 0.40 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 725-------------------- +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4712 iter count: 1.0 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4712 iter count: 1.0 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4712 iter count: 1.0 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4712iter count: 1.0 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:28:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662501 37.775776 37.583195 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656250 64.699219 64.527344 +util 96.250000 99.000000 90.000000 +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 726 : avg iteration time 47.20 seconds +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 726 : data loading time 0.00 +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 726 : avg forward pass time 12.68 +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 726 : backward pass time 10.13 +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 726 : optimizer step time 1.56 +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 726-------------------- +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:29:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662621 37.776287 37.583187 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.631836 64.660156 64.546875 +util 96.250000 100.000000 92.000000 +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 727 : avg iteration time 45.95 seconds +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 727 : data loading time 0.00 +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 727 : avg forward pass time 12.04 +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 727 : backward pass time 10.14 +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 727 : optimizer step time 1.57 +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 727-------------------- +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:30:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662683 37.776417 37.583302 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.666016 64.718750 64.507812 +util 93.750000 100.000000 88.000000 +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 728 : avg iteration time 45.92 seconds +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 728 : data loading time 0.00 +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 728 : avg forward pass time 12.01 +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 728 : backward pass time 10.15 +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 728 : optimizer step time 1.57 +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 728-------------------- +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:31:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662721 37.776421 37.583321 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.636719 64.660156 64.566406 +util 95.750000 100.000000 89.000000 +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 729 : avg iteration time 45.92 seconds +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 729 : data loading time 0.00 +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 729 : avg forward pass time 12.03 +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 729 : backward pass time 10.15 +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 729 : optimizer step time 1.57 +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 729-------------------- +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:31:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662537 37.775558 37.583324 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.244629 105.263672 105.238281 +util 64.000000 72.000000 58.000000 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 730 : avg iteration time 160.68 seconds +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 730 : data loading time 0.00 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 730 : avg forward pass time 71.76 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 730 : backward pass time 8.41 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 730 : optimizer step time 0.40 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 730-------------------- +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5137 iter count: 1.0 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5137 iter count: 1.0 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5137 iter count: 1.0 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5137iter count: 1.0 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:34:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662689 37.775951 37.583363 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.719727 64.781250 64.699219 +util 92.750000 99.000000 87.000000 +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 731 : avg iteration time 49.79 seconds +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 731 : data loading time 0.00 +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 731 : avg forward pass time 13.96 +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 731 : backward pass time 10.15 +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 731 : optimizer step time 1.57 +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 731-------------------- +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:35:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662831 37.776424 37.583408 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700195 64.703125 64.699219 +util 95.250000 99.000000 89.000000 +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 732 : avg iteration time 45.93 seconds +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 732 : data loading time 0.00 +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 732 : avg forward pass time 12.02 +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 732 : backward pass time 10.14 +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 732 : optimizer step time 1.58 +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 732-------------------- +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:36:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663116 37.777405 37.583363 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.568359 64.644531 64.542969 +util 93.500000 99.000000 89.000000 +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 733 : avg iteration time 43.52 seconds +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 733 : data loading time 0.00 +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 733 : avg forward pass time 10.81 +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 733 : backward pass time 10.13 +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 733 : optimizer step time 1.57 +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 733-------------------- +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:36:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663130 37.777416 37.583408 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.646484 64.722656 64.621094 +util 95.500000 99.000000 90.000000 +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 734 : avg iteration time 48.54 seconds +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 734 : data loading time 0.00 +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 734 : avg forward pass time 13.33 +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 734 : backward pass time 10.16 +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 734 : optimizer step time 1.57 +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 734-------------------- +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:37:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662892 37.776474 37.583420 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.864746 105.882812 105.810547 +util 67.000000 73.000000 60.000000 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 735 : avg iteration time 163.22 seconds +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 735 : data loading time 0.00 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 735 : avg forward pass time 72.97 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 735 : backward pass time 8.42 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 735 : optimizer step time 0.39 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 735-------------------- +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5303 iter count: 1.0 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5303 iter count: 1.0 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5303 iter count: 1.0 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5303iter count: 1.0 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:40:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662932 37.776440 37.583527 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.937500 64.777344 +util 95.250000 99.000000 91.000000 +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 736 : avg iteration time 49.87 seconds +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 736 : data loading time 0.00 +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 736 : avg forward pass time 14.00 +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 736 : backward pass time 10.15 +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 736 : optimizer step time 1.57 +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 736-------------------- +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:41:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663182 37.777447 37.583542 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.724609 64.859375 64.679688 +util 96.250000 99.000000 93.000000 +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 737 : avg iteration time 43.42 seconds +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 737 : data loading time 0.00 +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 737 : avg forward pass time 10.77 +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 737 : backward pass time 10.15 +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 737 : optimizer step time 1.57 +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 737-------------------- +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:42:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663327 37.777950 37.583595 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.685547 64.878906 64.621094 +util 97.250000 100.000000 91.000000 +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 738 : avg iteration time 46.05 seconds +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 738 : data loading time 0.00 +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 738 : avg forward pass time 12.09 +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 738 : backward pass time 10.17 +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 738 : optimizer step time 1.56 +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 738-------------------- +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:42:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663328 37.777950 37.583614 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.597656 64.660156 64.410156 +util 95.500000 100.000000 89.000000 +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 739 : avg iteration time 48.63 seconds +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 739 : data loading time 0.00 +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 739 : avg forward pass time 13.36 +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 739 : backward pass time 10.16 +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 739 : optimizer step time 1.58 +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 739-------------------- +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:43:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662908 37.776108 37.583599 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.210449 105.302734 105.179688 +util 65.500000 71.000000 60.000000 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 740 : avg iteration time 157.90 seconds +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 740 : data loading time 0.00 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 740 : avg forward pass time 70.37 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 740 : backward pass time 8.40 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 740 : optimizer step time 0.39 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 740-------------------- +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5200 iter count: 1.0 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5200 iter count: 1.0 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5200 iter count: 1.0 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5200iter count: 1.0 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:46:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662930 37.776115 37.583595 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.519531 64.523438 64.507812 +util 94.500000 99.000000 88.000000 +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 741 : avg iteration time 49.62 seconds +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 741 : data loading time 0.00 +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 741 : avg forward pass time 13.88 +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 741 : backward pass time 10.13 +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 741 : optimizer step time 1.57 +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 741-------------------- +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:47:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663124 37.776890 37.583595 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.509766 64.527344 64.503906 +util 94.000000 99.000000 90.000000 +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 742 : avg iteration time 45.94 seconds +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 742 : data loading time 0.00 +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 742 : avg forward pass time 12.05 +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 742 : backward pass time 10.15 +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 742 : optimizer step time 1.57 +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 742-------------------- +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:47:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663308 37.777618 37.583595 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656250 64.996094 64.542969 +util 96.000000 100.000000 90.000000 +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 743 : avg iteration time 48.54 seconds +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 743 : data loading time 0.00 +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 743 : avg forward pass time 13.33 +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 743 : backward pass time 10.16 +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 743 : optimizer step time 1.57 +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 743-------------------- +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:48:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663311 37.777618 37.583595 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.529297 64.542969 64.488281 +util 95.750000 99.000000 92.000000 +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 744 : avg iteration time 43.69 seconds +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 744 : data loading time 0.00 +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 744 : avg forward pass time 10.92 +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 744 : backward pass time 10.14 +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 744 : optimizer step time 1.56 +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 744-------------------- +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:49:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662953 37.776157 37.583603 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.254395 105.257812 105.244141 +util 68.000000 74.000000 63.000000 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 745 : avg iteration time 160.43 seconds +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 745 : data loading time 0.00 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 745 : avg forward pass time 71.62 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 745 : backward pass time 8.40 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 745 : optimizer step time 0.39 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 745-------------------- +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4937 iter count: 1.0 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4937 iter count: 1.0 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4937 iter count: 1.0 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4937iter count: 1.0 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:52:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.662967 37.776161 37.583618 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.817383 64.937500 64.777344 +util 92.000000 98.000000 87.000000 +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 746 : avg iteration time 47.12 seconds +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 746 : data loading time 0.00 +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 746 : avg forward pass time 12.62 +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 746 : backward pass time 10.15 +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 746 : optimizer step time 1.57 +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 746-------------------- +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:52:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663032 37.776421 37.583618 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.792969 64.957031 64.738281 +util 94.500000 100.000000 89.000000 +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 747 : avg iteration time 45.99 seconds +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 747 : data loading time 0.00 +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 747 : avg forward pass time 12.05 +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 747 : backward pass time 10.15 +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 747 : optimizer step time 1.58 +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 747-------------------- +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:53:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663417 37.777939 37.583618 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.695312 64.859375 64.640625 +util 92.750000 98.000000 88.000000 +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 748 : avg iteration time 46.03 seconds +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 748 : data loading time 0.00 +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 748 : avg forward pass time 12.09 +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 748 : backward pass time 10.14 +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 748 : optimizer step time 1.56 +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 748-------------------- +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:54:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663454 37.778015 37.583660 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.685547 64.757812 64.468750 +util 96.750000 99.000000 94.000000 +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 749 : avg iteration time 46.06 seconds +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 749 : data loading time 0.00 +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 749 : avg forward pass time 12.09 +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 749 : backward pass time 10.15 +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 749 : optimizer step time 1.57 +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 749-------------------- +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:55:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663483 37.778118 37.583645 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.312988 105.335938 105.244141 +util 61.500000 68.000000 56.000000 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 750 : avg iteration time 163.33 seconds +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 750 : data loading time 0.00 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 750 : avg forward pass time 73.06 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 750 : backward pass time 8.44 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 750 : optimizer step time 0.39 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 750-------------------- +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5220 iter count: 1.0 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5220 iter count: 1.0 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5220 iter count: 1.0 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5220iter count: 1.0 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:57:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663288 37.777325 37.583645 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.841797 64.917969 64.816406 +util 96.500000 100.000000 91.000000 +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 751 : avg iteration time 44.77 seconds +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 751 : data loading time 0.00 +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 751 : avg forward pass time 11.45 +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 751 : backward pass time 10.14 +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 751 : optimizer step time 1.56 +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 751-------------------- +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:58:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663332 37.777328 37.583790 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.763672 65.015625 64.679688 +util 94.750000 99.000000 88.000000 +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 752 : avg iteration time 43.32 seconds +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 752 : data loading time 0.00 +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 752 : avg forward pass time 10.73 +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 752 : backward pass time 10.16 +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 752 : optimizer step time 1.56 +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 752-------------------- +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 17:59:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663361 37.777309 37.583935 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.744141 64.996094 64.660156 +util 94.000000 99.000000 87.000000 +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 753 : avg iteration time 46.04 seconds +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 753 : data loading time 0.00 +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 753 : avg forward pass time 12.09 +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 753 : backward pass time 10.14 +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 753 : optimizer step time 1.56 +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 753-------------------- +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:00:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663364 37.777309 37.583935 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734375 64.957031 64.660156 +util 93.500000 99.000000 89.000000 +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 754 : avg iteration time 43.29 seconds +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 754 : data loading time 0.00 +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 754 : avg forward pass time 10.70 +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 754 : backward pass time 10.15 +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 754 : optimizer step time 1.57 +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 754-------------------- +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:00:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663461 37.777580 37.583935 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.269043 105.277344 105.244141 +util 66.500000 72.000000 60.000000 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 755 : avg iteration time 163.50 seconds +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 755 : data loading time 0.00 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 755 : avg forward pass time 73.16 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 755 : backward pass time 8.41 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 755 : optimizer step time 0.40 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 755-------------------- +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5459 iter count: 1.0 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5459 iter count: 1.0 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5459 iter count: 1.0 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5459iter count: 1.0 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:03:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663445 37.777416 37.583935 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.749512 64.958984 64.679688 +util 93.500000 99.000000 88.000000 +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 756 : avg iteration time 49.65 seconds +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 756 : data loading time 0.00 +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 756 : avg forward pass time 13.89 +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 756 : backward pass time 10.16 +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 756 : optimizer step time 1.56 +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 756-------------------- +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:04:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663468 37.777512 37.583935 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.734863 64.738281 64.724609 +util 96.750000 99.000000 93.000000 +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 757 : avg iteration time 43.30 seconds +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 757 : data loading time 0.00 +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 757 : avg forward pass time 10.71 +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 757 : backward pass time 10.16 +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 757 : optimizer step time 1.57 +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 757-------------------- +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:05:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663470 37.777512 37.583935 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.681152 64.685547 64.679688 +util 94.000000 99.000000 87.000000 +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 758 : avg iteration time 43.25 seconds +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 758 : data loading time 0.00 +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 758 : avg forward pass time 10.69 +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 758 : backward pass time 10.14 +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 758 : optimizer step time 1.57 +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 758-------------------- +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:05:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663714 37.778511 37.583900 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.666504 64.744141 64.640625 +util 95.000000 99.000000 89.000000 +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 759 : avg iteration time 48.62 seconds +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 759 : data loading time 0.00 +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 759 : avg forward pass time 13.37 +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 759 : backward pass time 10.15 +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 759 : optimizer step time 1.57 +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 759-------------------- +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:06:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663494 37.777592 37.583950 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.342285 105.771484 105.199219 +util 65.250000 71.000000 59.000000 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 760 : avg iteration time 158.36 seconds +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 760 : data loading time 0.00 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 760 : avg forward pass time 70.59 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 760 : backward pass time 8.40 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 760 : optimizer step time 0.39 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 760-------------------- +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4912 iter count: 1.0 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4912 iter count: 1.0 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4912 iter count: 1.0 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4912iter count: 1.0 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:09:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663559 37.777843 37.583958 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.612305 64.820312 64.542969 +util 94.500000 99.000000 88.000000 +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 761 : avg iteration time 49.58 seconds +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 761 : data loading time 0.00 +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 761 : avg forward pass time 13.86 +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 761 : backward pass time 10.14 +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 761 : optimizer step time 1.58 +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 761-------------------- +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:10:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663562 37.777851 37.583961 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656250 64.703125 64.640625 +util 94.750000 99.000000 88.000000 +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 762 : avg iteration time 43.26 seconds +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 762 : data loading time 0.00 +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 762 : avg forward pass time 10.70 +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 762 : backward pass time 10.14 +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 762 : optimizer step time 1.56 +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 762-------------------- +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:10:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663568 37.777859 37.583961 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.636719 64.742188 64.601562 +util 95.000000 99.000000 89.000000 +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 763 : avg iteration time 48.55 seconds +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 763 : data loading time 0.00 +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 763 : avg forward pass time 13.35 +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 763 : backward pass time 10.14 +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 763 : optimizer step time 1.56 +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 763-------------------- +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:11:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663772 37.778645 37.583961 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.558594 64.722656 64.503906 +util 92.500000 99.000000 87.000000 +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 764 : avg iteration time 45.92 seconds +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 764 : data loading time 0.00 +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 764 : avg forward pass time 12.02 +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 764 : backward pass time 10.15 +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 764 : optimizer step time 1.57 +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 764-------------------- +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:12:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663670 37.778057 37.583965 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.205566 105.224609 105.199219 +util 67.500000 73.000000 62.000000 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 765 : avg iteration time 163.13 seconds +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 765 : data loading time 0.00 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 765 : avg forward pass time 72.98 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 765 : backward pass time 8.38 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 765 : optimizer step time 0.39 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 765-------------------- +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5693 iter count: 1.0 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5693 iter count: 1.0 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5693 iter count: 1.0 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5693iter count: 1.0 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:15:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663684 37.778065 37.583988 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.724609 64.742188 64.718750 +util 96.000000 100.000000 92.000000 +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 766 : avg iteration time 44.53 seconds +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 766 : data loading time 0.00 +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 766 : avg forward pass time 11.33 +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 766 : backward pass time 10.14 +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 766 : optimizer step time 1.57 +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 766-------------------- +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:15:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663713 37.778065 37.584015 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.676270 64.699219 64.607422 +util 96.750000 100.000000 91.000000 +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 767 : avg iteration time 45.86 seconds +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 767 : data loading time 0.00 +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 767 : avg forward pass time 12.00 +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 767 : backward pass time 10.14 +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 767 : optimizer step time 1.56 +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 767-------------------- +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:16:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663735 37.778072 37.584019 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.661621 64.679688 64.607422 +util 91.750000 98.000000 86.000000 +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 768 : avg iteration time 48.48 seconds +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 768 : data loading time 0.00 +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 768 : avg forward pass time 13.32 +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 768 : backward pass time 10.14 +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 768 : optimizer step time 1.56 +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 768-------------------- +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:17:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663769 37.778072 37.584023 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.666504 64.679688 64.626953 +util 91.750000 98.000000 86.000000 +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 769 : avg iteration time 46.00 seconds +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 769 : data loading time 0.00 +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 769 : avg forward pass time 12.08 +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 769 : backward pass time 10.13 +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 769 : optimizer step time 1.56 +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 769-------------------- +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:18:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663778 37.777954 37.584026 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.322754 105.355469 105.224609 +util 65.250000 71.000000 60.000000 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 770 : avg iteration time 160.66 seconds +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 770 : data loading time 0.00 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 770 : avg forward pass time 71.74 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 770 : backward pass time 8.40 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 770 : optimizer step time 0.39 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 770-------------------- +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5361 iter count: 1.0 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5361 iter count: 1.0 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5361 iter count: 1.0 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5361iter count: 1.0 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:20:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663834 37.777962 37.584026 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700195 64.878906 64.640625 +util 93.500000 100.000000 87.000000 +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 771 : avg iteration time 44.32 seconds +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 771 : data loading time 0.00 +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 771 : avg forward pass time 11.23 +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 771 : backward pass time 10.14 +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 771 : optimizer step time 1.57 +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 771-------------------- +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:21:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663838 37.777966 37.584030 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.705078 64.957031 64.621094 +util 94.000000 99.000000 87.000000 +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 772 : avg iteration time 48.56 seconds +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 772 : data loading time 0.00 +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 772 : avg forward pass time 13.36 +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 772 : backward pass time 10.13 +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 772 : optimizer step time 1.57 +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 772-------------------- +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:22:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663833 37.777969 37.584034 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.636719 64.917969 64.542969 +util 96.750000 100.000000 91.000000 +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 773 : avg iteration time 43.39 seconds +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 773 : data loading time 0.00 +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 773 : avg forward pass time 10.75 +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 773 : backward pass time 10.16 +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 773 : optimizer step time 1.56 +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 773-------------------- +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:23:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663915 37.778011 37.584034 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.651367 64.917969 64.562500 +util 94.000000 99.000000 89.000000 +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 774 : avg iteration time 48.51 seconds +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 774 : data loading time 0.00 +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 774 : avg forward pass time 13.31 +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 774 : backward pass time 10.15 +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 774 : optimizer step time 1.58 +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 774-------------------- +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:23:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663599 37.776718 37.584042 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.293457 105.296875 105.283203 +util 65.750000 72.000000 60.000000 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 775 : avg iteration time 165.63 seconds +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 775 : data loading time 0.00 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 775 : avg forward pass time 74.24 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 775 : backward pass time 8.38 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 775 : optimizer step time 0.39 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 775-------------------- +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5654 iter count: 1.0 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5654 iter count: 1.0 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5654 iter count: 1.0 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5654iter count: 1.0 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:26:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663588 37.776718 37.584042 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.720215 64.900391 64.660156 +util 95.000000 99.000000 89.000000 +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 776 : avg iteration time 49.63 seconds +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 776 : data loading time 0.00 +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 776 : avg forward pass time 13.88 +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 776 : backward pass time 10.14 +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 776 : optimizer step time 1.57 +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 776-------------------- +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:27:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663796 37.777435 37.584042 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.759277 64.880859 64.718750 +util 93.750000 100.000000 88.000000 +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 777 : avg iteration time 45.88 seconds +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 777 : data loading time 0.00 +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 777 : avg forward pass time 12.00 +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 777 : backward pass time 10.16 +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 777 : optimizer step time 1.57 +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 777-------------------- +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:28:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663852 37.777493 37.584087 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.764160 64.900391 64.718750 +util 94.500000 99.000000 88.000000 +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 778 : avg iteration time 43.37 seconds +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 778 : data loading time 0.00 +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 778 : avg forward pass time 10.74 +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 778 : backward pass time 10.16 +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 778 : optimizer step time 1.57 +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 778-------------------- +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:29:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663940 37.777763 37.584129 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.769043 64.919922 64.718750 +util 93.250000 99.000000 88.000000 +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 779 : avg iteration time 45.88 seconds +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 779 : data loading time 0.00 +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 779 : avg forward pass time 12.01 +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 779 : backward pass time 10.14 +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 779 : optimizer step time 1.57 +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 779-------------------- +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:29:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663766 37.776890 37.584087 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.278809 105.296875 105.224609 +util 61.000000 68.000000 55.000000 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 780 : avg iteration time 163.39 seconds +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 780 : data loading time 0.00 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 780 : avg forward pass time 73.13 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 780 : backward pass time 8.39 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 780 : optimizer step time 0.39 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 780-------------------- +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4639 iter count: 1.0 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4639 iter count: 1.0 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4639 iter count: 1.0 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4639iter count: 1.0 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:32:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663866 37.777195 37.584110 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.871094 64.976562 64.835938 +util 95.750000 100.000000 90.000000 +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 781 : avg iteration time 47.16 seconds +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 781 : data loading time 0.00 +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 781 : avg forward pass time 12.65 +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 781 : backward pass time 10.15 +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 781 : optimizer step time 1.56 +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 781-------------------- +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:33:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664074 37.777946 37.584110 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.768555 64.976562 64.699219 +util 95.250000 99.000000 89.000000 +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 782 : avg iteration time 45.94 seconds +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 782 : data loading time 0.00 +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 782 : avg forward pass time 12.02 +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 782 : backward pass time 10.18 +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 782 : optimizer step time 1.57 +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 782-------------------- +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:34:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664153 37.778191 37.584110 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.587891 64.601562 64.546875 +util 95.000000 100.000000 88.000000 +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 783 : avg iteration time 43.25 seconds +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 783 : data loading time 0.00 +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 783 : avg forward pass time 10.71 +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 783 : backward pass time 10.15 +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 783 : optimizer step time 1.55 +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 783-------------------- +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:34:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664211 37.778191 37.584133 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.656250 64.878906 64.582031 +util 95.250000 99.000000 91.000000 +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 784 : avg iteration time 45.91 seconds +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 784 : data loading time 0.00 +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 784 : avg forward pass time 12.01 +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 784 : backward pass time 10.15 +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 784 : optimizer step time 1.57 +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 784-------------------- +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:35:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663967 37.777248 37.584110 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.244629 105.263672 105.238281 +util 65.250000 71.000000 58.000000 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 785 : avg iteration time 162.91 seconds +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 785 : data loading time 0.00 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 785 : avg forward pass time 72.88 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 785 : backward pass time 8.39 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 785 : optimizer step time 0.39 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 785-------------------- +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5371 iter count: 1.0 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5371 iter count: 1.0 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5371 iter count: 1.0 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5371iter count: 1.0 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:38:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663968 37.777252 37.584110 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.890625 64.972656 64.644531 +util 94.750000 99.000000 88.000000 +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 786 : avg iteration time 44.39 seconds +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 786 : data loading time 0.00 +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 786 : avg forward pass time 11.27 +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 786 : backward pass time 10.15 +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 786 : optimizer step time 1.57 +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 786-------------------- +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:39:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.663992 37.777271 37.584087 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 65.011719 64.566406 +util 97.000000 100.000000 91.000000 +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 787 : avg iteration time 45.98 seconds +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 787 : data loading time 0.00 +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 787 : avg forward pass time 12.07 +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 787 : backward pass time 10.14 +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 787 : optimizer step time 1.56 +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 787-------------------- +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:39:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664453 37.779022 37.584087 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.880859 64.972656 64.605469 +util 94.000000 99.000000 89.000000 +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 788 : avg iteration time 45.91 seconds +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 788 : data loading time 0.00 +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 788 : avg forward pass time 12.03 +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 788 : backward pass time 10.15 +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 788 : optimizer step time 1.56 +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 788-------------------- +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:40:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664579 37.779522 37.584087 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.866211 64.933594 64.664062 +util 95.500000 99.000000 91.000000 +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 789 : avg iteration time 46.02 seconds +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 789 : data loading time 0.00 +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 789 : avg forward pass time 12.08 +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 789 : backward pass time 10.14 +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 789 : optimizer step time 1.56 +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 789-------------------- +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:41:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664376 37.778687 37.584087 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.215332 105.263672 105.199219 +util 64.500000 70.000000 59.000000 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 790 : avg iteration time 160.45 seconds +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 790 : data loading time 0.00 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 790 : avg forward pass time 71.66 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 790 : backward pass time 8.38 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 790 : optimizer step time 0.40 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 790-------------------- +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.5469 iter count: 1.0 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5469 iter count: 1.0 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.5469 iter count: 1.0 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5469iter count: 1.0 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:43:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664303 37.778393 37.584087 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.836914 64.894531 64.664062 +util 96.750000 100.000000 91.000000 +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 791 : avg iteration time 47.02 seconds +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 791 : data loading time 0.00 +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 791 : avg forward pass time 12.58 +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 791 : backward pass time 10.14 +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 791 : optimizer step time 1.57 +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 791-------------------- +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:44:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664309 37.778397 37.584087 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.963867 65.050781 64.703125 +util 94.000000 99.000000 88.000000 +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 792 : avg iteration time 45.96 seconds +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 792 : data loading time 0.00 +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 792 : avg forward pass time 12.04 +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 792 : backward pass time 10.15 +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 792 : optimizer step time 1.57 +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 792-------------------- +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:45:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664373 37.778637 37.584106 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.900391 64.972656 64.683594 +util 93.750000 99.000000 89.000000 +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 793 : avg iteration time 43.29 seconds +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 793 : data loading time 0.00 +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 793 : avg forward pass time 10.71 +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 793 : backward pass time 10.15 +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 793 : optimizer step time 1.57 +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 793-------------------- +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:46:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664443 37.778893 37.584106 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.895508 64.953125 64.722656 +util 95.500000 100.000000 89.000000 +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 794 : avg iteration time 43.39 seconds +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 794 : data loading time 0.00 +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 794 : avg forward pass time 10.76 +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 794 : backward pass time 10.14 +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 794 : optimizer step time 1.57 +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 794-------------------- +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:46:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664488 37.778900 37.584225 +peak_gpu_mem_gb 98.769184 98.772308 98.768142 +peak_gpu_mem_reserved_gb 105.200684 105.205078 105.199219 +util 64.250000 68.000000 62.000000 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 795 : avg iteration time 157.92 seconds +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 795 : data loading time 0.00 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 795 : avg forward pass time 70.38 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 795 : backward pass time 8.38 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 795 : optimizer step time 0.39 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 795-------------------- +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.4863 iter count: 1.0 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4863 iter count: 1.0 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 0.4863 iter count: 1.0 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4863iter count: 1.0 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:49:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664535 37.778797 37.584244 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.690430 64.898438 64.621094 +util 95.500000 100.000000 89.000000 +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 796 : avg iteration time 44.28 seconds +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 796 : data loading time 0.00 +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 796 : avg forward pass time 11.23 +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 796 : backward pass time 10.11 +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 796 : optimizer step time 1.56 +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 796-------------------- +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:50:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664620 37.779064 37.584232 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.685547 64.878906 64.621094 +util 95.250000 99.000000 89.000000 +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 797 : avg iteration time 46.00 seconds +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 797 : data loading time 0.00 +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 797 : avg forward pass time 12.07 +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 797 : backward pass time 10.14 +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 797 : optimizer step time 1.57 +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 797-------------------- +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:51:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664624 37.779064 37.584232 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.700195 64.937500 64.621094 +util 93.000000 99.000000 88.000000 +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 798 : avg iteration time 45.87 seconds +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 798 : data loading time 0.00 +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 798 : avg forward pass time 11.99 +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 798 : backward pass time 10.17 +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 798 : optimizer step time 1.56 +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 798-------------------- +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:51:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.664640 37.779087 37.584251 +peak_gpu_mem_gb 62.750208 62.753333 62.749167 +peak_gpu_mem_reserved_gb 64.675781 64.839844 64.621094 +util 94.750000 100.000000 90.000000 +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 799 : avg iteration time 43.23 seconds +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 799 : data loading time 0.00 +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 799 : avg forward pass time 10.69 +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 799 : backward pass time 10.14 +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 799 : optimizer step time 1.57 +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 799-------------------- +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 18:52:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 18:52:35 | INFO | fastgen.trainer:run:225 ] Training complete. +[May 5, 2026 - 18:52:35 | INFO | fastgen.trainer:validate:358 ] Validation iteration 800 +[May 5, 2026 - 18:52:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:52:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.4s +[May 5, 2026 - 18:52:43 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 5, 2026 - 18:52:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.195 +[May 5, 2026 - 18:52:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:52:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 18:52:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.351 +[May 5, 2026 - 18:52:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:52:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.7s +[May 5, 2026 - 18:52:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.562 +[May 5, 2026 - 18:52:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:52:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 5, 2026 - 18:52:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.450 +[May 5, 2026 - 18:52:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:53:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 5, 2026 - 18:53:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.241 +[May 5, 2026 - 18:53:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:53:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 5, 2026 - 18:53:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.086 +[May 5, 2026 - 18:53:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:53:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 5, 2026 - 18:53:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.102 +[May 5, 2026 - 18:53:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:53:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 5, 2026 - 18:53:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.836 +[May 5, 2026 - 18:53:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:53:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 5, 2026 - 18:53:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.797 +[May 5, 2026 - 18:53:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 5, 2026 - 18:53:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.6s +[May 5, 2026 - 18:53:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.820 +[May 5, 2026 - 18:53:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 800-------------------- +[May 5, 2026 - 18:53:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 5, 2026 - 18:53:45 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 800 +[May 5, 2026 - 18:53:45 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.444 (n=10) +[May 5, 2026 - 18:53:45 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 800 +[May 5, 2026 - 18:53:45 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints/0000800 +[May 5, 2026 - 18:54:01 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 800 +[May 5, 2026 - 18:54:02 | INFO | fastgen.trainer:run:232 ] Taking a 10 sec nap and exiting training. +[May 5, 2026 - 18:54:12 | INFO | fastgen.utils.distributed:clean_up:88 ] Distributed clean up. +[May 5, 2026 - 18:54:12 | INFO | __main__::49 ] Training finished. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9c5e8006e0eb3baa9927e88507b36749d6616bd6 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-05T07:57:00.623568Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769_no_reward.py", + "-", + "trainer.resume=True", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769_no_reward", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=800", + "trainer.save_ckpt_iter=100" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1176060735488" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "ijbznehtvc6ciol7xuxzz1ktw5k81eu8" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5f7d2f496e0d7483298ededb5cae48a5950477bc --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/files/wandb-summary.json @@ -0,0 +1 @@ +{"profiler/avg_forward_pass_time":10.69363664649427,"profiler/optimizer_step_time":1.5700227953493595,"optimizer/lr_fake_score":2e-06,"optimizer/non_finite_grads_count (model_key net)":0,"fake_score/trainable_params":1421383840,"val0/generated":{"_type":"videos","count":10,"videos":[{"sha256":"56f78017d0746eb5d2dc188735b0cf6938615b98c2a11286b10826e83c91fb35","size":216854,"path":"media/videos/val0/generated_800_56f78017d0746eb5d2dc.mp4","caption":"sync_c=6.195","_type":"video-file"},{"_type":"video-file","sha256":"6853c5e28978367eb479667a6d60b6fa991de2e9ee5c993d42f2654567917be2","size":219627,"path":"media/videos/val0/generated_800_6853c5e28978367eb479.mp4","caption":"sync_c=6.351"},{"path":"media/videos/val0/generated_800_59adee4c2c1233294521.mp4","caption":"sync_c=5.562","_type":"video-file","sha256":"59adee4c2c12332945212a9f37785447fde66ae3136b437df8ca1975a263c091","size":210861},{"_type":"video-file","sha256":"b2a3a97608a2bdd39582fba0c457a16625d74b22ddb8a771b9fc0c2ca0225a1b","size":302581,"path":"media/videos/val0/generated_800_b2a3a97608a2bdd39582.mp4","caption":"sync_c=5.450"},{"size":260341,"path":"media/videos/val0/generated_800_b4cb8b6f832fd8a17929.mp4","caption":"sync_c=4.241","_type":"video-file","sha256":"b4cb8b6f832fd8a179298dc8b3f479408dd7f3ddd21aadad7cf2c64414218e79"},{"sha256":"dfe41c776e29223e9b8cef0d4076ba721edf4dd6f0a6b2566f3b0769f9070ab2","size":192865,"path":"media/videos/val0/generated_800_dfe41c776e29223e9b8c.mp4","caption":"sync_c=7.086","_type":"video-file"},{"_type":"video-file","sha256":"75bb737f23f37c356994ec54519c0b7eb03e9879031d9093cfdab75250db3c2c","size":301276,"path":"media/videos/val0/generated_800_75bb737f23f37c356994.mp4","caption":"sync_c=6.102"},{"path":"media/videos/val0/generated_800_8180e794591d55c93416.mp4","caption":"sync_c=4.836","_type":"video-file","sha256":"8180e794591d55c93416e298518e6c3b53acab661fd0b2a3ec3fdcc61059b98b","size":200578},{"_type":"video-file","sha256":"18cdb0fb0acea943ae1be8bc98f8554d890bed7d4c5686dd660c3e3ef3567026","size":243732,"path":"media/videos/val0/generated_800_18cdb0fb0acea943ae1b.mp4","caption":"sync_c=5.797"},{"caption":"sync_c=2.820","_type":"video-file","sha256":"85b5b43e4c471767517fa55c958b4d0bbcee7d66b96b35efd6cef44a3a11d3d1","size":246015,"path":"media/videos/val0/generated_800_85b5b43e4c471767517f.mp4"}],"captions":["sync_c=6.195","sync_c=6.351","sync_c=5.562","sync_c=5.450","sync_c=4.241","sync_c=7.086","sync_c=6.102","sync_c=4.836","sync_c=5.797","sync_c=2.820"]},"model/local_trainable_params":[711111248,710552144,710552144,710552144],"model/trainable_params":2842767680,"train/total_loss":0.0106201171875,"train/vsd_loss":0.486328125,"_step":800,"train_media/student/generation":{"sha256":"54d7e0b7b8631130c0130dc593aca13885eead1f40bb955833b5027567d55eaa","size":215988,"_type":"video-file","path":"media/videos/train_media/student/generation_500_54d7e0b7b8631130c013.mp4"},"val0/sync_c_sample_8":5.797165393829346,"model/local_total_params":[4285560952,4283762808,4283762808,4283762808],"fake_score/local_trainable_params":[355555624,355276072,355276072,355276072],"val0/sync_c_sample_6":6.10244607925415,"net/local_trainable_params":[355555624,355276072,355276072,355276072],"optimizer/grad_norm (model_key net)":0,"val0/reconstructed":{"count":10,"videos":[{"path":"media/videos/val0/reconstructed_800_80aab00fcfa85d5db98e.mp4","_type":"video-file","sha256":"80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327","size":182028},{"size":170922,"path":"media/videos/val0/reconstructed_800_884b2d197a5511393680.mp4","_type":"video-file","sha256":"884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214"},{"size":173236,"path":"media/videos/val0/reconstructed_800_c0196720a45889bb2198.mp4","_type":"video-file","sha256":"c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8"},{"size":249372,"path":"media/videos/val0/reconstructed_800_294040ac378ceaa34329.mp4","_type":"video-file","sha256":"294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c"},{"path":"media/videos/val0/reconstructed_800_15ac5b765a401d035db1.mp4","_type":"video-file","sha256":"15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117","size":190004},{"_type":"video-file","sha256":"f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395","size":164630,"path":"media/videos/val0/reconstructed_800_f5463a4ea6f68a39c5d0.mp4"},{"_type":"video-file","sha256":"c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8","size":222986,"path":"media/videos/val0/reconstructed_800_c0f892b5fba78348af85.mp4"},{"size":165516,"path":"media/videos/val0/reconstructed_800_a076d17e202ed18ebdf8.mp4","_type":"video-file","sha256":"a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb"},{"sha256":"6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac","size":214001,"path":"media/videos/val0/reconstructed_800_6ff046418b5d41477989.mp4","_type":"video-file"},{"_type":"video-file","sha256":"f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273","size":227490,"path":"media/videos/val0/reconstructed_800_f7412c9e170ffd299748.mp4"}],"captions":false,"_type":"videos"},"fake_score/total_params":1421383840,"_timestamp":1.7779748252730575e+09,"val0/sync_c_sample_5":7.085724830627441,"val0/sync_c_sample_2":5.561923503875732,"train/vsd_loss_unweighted":0.486328125,"train/gan_loss_gen":0,"optimizer/iteration":800,"fake_score/local_total_params":[355555624,355276072,355276072,355276072],"val0/total_loss":0,"model/total_params":17136849376,"val0/sync_c_sample_3":5.449801445007324,"train_media/data/real":{"path":"media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4","sha256":"2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae","size":190461,"_type":"video-file"},"train/vsd_loss_weighted":0.486328125,"val0/sync_c_sample_9":2.8201112747192383,"val0/sync_c_mean":5.44401364326477,"profiler/data_loading_time":0.0008579939603805542,"profiler/avg_iteration_time":43.23180818557739,"val0/sync_c_sample_0":6.194889545440674,"train/fake_score_loss":0.0106201171875,"optimizer/lr_net":2e-06,"val0/sync_c_sample_4":4.2412309646606445,"val0/sync_c_sample_7":4.83624267578125,"_runtime":56986.476234612,"train/gan_loss_disc":0,"profiler/backward_pass_time":10.137909101322293,"_wandb":{"runtime":56986},"net/trainable_params":1421383840,"val0/sync_c_sample_1":6.350600719451904,"net/total_params":1421383840,"net/local_total_params":[355555624,355276072,355276072,355276072]} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d05995960451775b2d9227af31a53e82b208a73e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-05-05T16:57:00.974277213+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpzlj1eapz/port-784820.txt","pid":784820,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-05T16:57:00.975802148+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-784820-799105-3586771523/socket","Net":"unix"}} +{"time":"2026-05-05T16:57:00.976315884+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":784820} +{"time":"2026-05-05T16:57:01.083228201+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-05T16:57:01.102700031+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lw2bm1l6","id":"1(@)"} +{"time":"2026-05-05T16:57:01.513192501+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lw2bm1l6","id":"1(@)"} +{"time":"2026-05-05T16:57:07.442077975+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"qsh70w61edno"} +{"time":"2026-05-05T18:54:12.660255926+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T18:54:12.660308896+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T18:54:12.660303987+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T18:54:12.660368396+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T18:54:12.66038611+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-784820-799105-3586771523/socket","Net":"unix"}} +{"time":"2026-05-05T18:54:16.419498555+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-05T18:54:16.419519227+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-05T18:54:16.419526635+09:00","level":"INFO","msg":"server is closed"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fb2cd06cc004ee799e3e0a5fba0e523ae3e25759 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T16:57:01.103690934+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T16:57:01.512285583+09:00","level":"INFO","msg":"stream: created new stream","id":"lw2bm1l6"} +{"time":"2026-05-05T16:57:01.513131599+09:00","level":"INFO","msg":"handler: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T16:57:01.513185968+09:00","level":"INFO","msg":"stream: started","id":"lw2bm1l6"} +{"time":"2026-05-05T16:57:01.513204736+09:00","level":"INFO","msg":"sender: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T16:57:01.513240232+09:00","level":"INFO","msg":"writer: started","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T18:54:12.660302666+09:00","level":"INFO","msg":"stream: closing","id":"lw2bm1l6"} +{"time":"2026-05-05T18:54:13.880823166+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T18:54:16.418725916+09:00","level":"INFO","msg":"handler: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T18:54:16.419293593+09:00","level":"INFO","msg":"sender: closed","stream_id":"lw2bm1l6"} +{"time":"2026-05-05T18:54:16.419315458+09:00","level":"INFO","msg":"stream: closed","id":"lw2bm1l6"} diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a56b6b2b4a4248ca1266c2f1b9f13e3a38af5f87 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug.log @@ -0,0 +1,22 @@ +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_setup.py:_flush():81] Configure stats pid to 784820 +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug.log +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/logs/debug-internal.log +2026-05-05 16:57:00,624 INFO MainThread:784820 [wandb_init.py:init():844] calling init triggers +2026-05-05 16:57:00,625 INFO MainThread:784820 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769_no_reward', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '800', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'True', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'False', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '0', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 16:57:00,625 INFO MainThread:784820 [wandb_init.py:init():892] starting backend +2026-05-05 16:57:01,082 INFO MainThread:784820 [wandb_init.py:init():895] sending inform_init request +2026-05-05 16:57:01,097 INFO MainThread:784820 [wandb_init.py:init():903] backend started and connected +2026-05-05 16:57:01,100 INFO MainThread:784820 [wandb_init.py:init():973] updated telemetry +2026-05-05 16:57:01,121 INFO MainThread:784820 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 16:57:02,180 INFO MainThread:784820 [wandb_init.py:init():1037] run resumed +2026-05-05 16:57:02,183 INFO MainThread:784820 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 16:57:02,436 INFO MainThread:784820 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 16:57:02,436 INFO MainThread:784820 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 16:57:02,436 INFO MainThread:784820 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 16:57:02,436 INFO MainThread:784820 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 16:57:02,441 INFO MainThread:784820 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 18:54:12,660 INFO wandb-AsyncioManager-main:784820 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 18:54:12,662 INFO wandb-AsyncioManager-main:784820 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/run-lw2bm1l6.wandb b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/run-lw2bm1l6.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f66181b4bd2934ed272e496748d131b8596b5343 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb/run-20260505_165700-lw2bm1l6/run-lw2bm1l6.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57aaafdb384bba1845e83726038ee61e2f3b33cff760afa5497ec5614090939e +size 1571617 diff --git a/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..d5e5634df9b8c67d5e17cccf6769e509b27b3bf9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_OUTPUT_FULL_FT_T769_NO_REWARD_CONTINUE_700_TO_800_SYMLINK/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769_no_reward/wandb_id.txt @@ -0,0 +1 @@ +lw2bm1l6 diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/config.yaml b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd9400f20e16345d300cda231f54aa4defd21caa --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/config.yaml @@ -0,0 +1,297 @@ +dataloader_train: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_audiofix + name: sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + lora_alpha: '64' + lora_rank: '128' + merge_lora: 'False' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + unfreeze_modules: + - model.audio_proj + - model.audio_cond_projs + - model.patch_embedding + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 4e-07 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'True' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + lora_alpha: '64' + lora_rank: '128' + merge_lora: 'False' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + unfreeze_modules: + - _core.audio_proj + - _core.audio_cond_projs + - _core.patch_embedding + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'True' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '2' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.769' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + reverse: 'False' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_14b_lora_t769_4gpu_bs4_grad1_eff16_lr1e5_5000iter/checkpoints/0003000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '100000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '4' + logging_iter: '1' + max_iter: '2500' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2061d7dbe2c6f030cc8a08711de12a118a77ff30 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T03:52:10.883134411+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T03:52:11.417276647+09:00","level":"INFO","msg":"stream: created new stream","id":"fxrufsq8"} +{"time":"2026-05-05T03:52:11.417813903+09:00","level":"INFO","msg":"handler: started","stream_id":"fxrufsq8"} +{"time":"2026-05-05T03:52:11.417896447+09:00","level":"INFO","msg":"stream: started","id":"fxrufsq8"} +{"time":"2026-05-05T03:52:11.417921032+09:00","level":"INFO","msg":"sender: started","stream_id":"fxrufsq8"} +{"time":"2026-05-05T03:52:11.417945996+09:00","level":"INFO","msg":"writer: started","stream_id":"fxrufsq8"} +{"time":"2026-05-05T04:20:59.201594537+09:00","level":"INFO","msg":"stream: closing","id":"fxrufsq8"} +{"time":"2026-05-05T04:20:59.821709487+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T04:21:00.158020868+09:00","level":"INFO","msg":"handler: closed","stream_id":"fxrufsq8"} +{"time":"2026-05-05T04:21:00.158092039+09:00","level":"INFO","msg":"sender: closed","stream_id":"fxrufsq8"} +{"time":"2026-05-05T04:21:00.158108163+09:00","level":"INFO","msg":"stream: closed","id":"fxrufsq8"} diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/debug.log b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b8ce0c7c15cec8690e89a35385d199e0c50cf9db --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/debug.log @@ -0,0 +1,21 @@ +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_setup.py:_flush():81] Configure stats pid to 3081796 +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug.log +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug-internal.log +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:init():844] calling init triggers +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_14b_lora_t769_4gpu_bs4_grad1_eff16_lr1e5_5000iter/checkpoints/0003000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '2500', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '4', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '100000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': "", 'merge_lora': 'False', 'unfreeze_modules': ['_core.audio_proj', '_core.audio_cond_projs', '_core.patch_embedding'], 'lora_rank': '128', 'lora_alpha': '64'}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "", 'unfreeze_modules': ['model.audio_proj', 'model.audio_cond_projs', 'model.patch_embedding'], 'lora_rank': '128', 'lora_alpha': '64'}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'True', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '4e-07', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:init():892] starting backend +2026-05-05 03:52:10,853 INFO MainThread:3081796 [wandb_init.py:init():895] sending inform_init request +2026-05-05 03:52:10,878 INFO MainThread:3081796 [wandb_init.py:init():903] backend started and connected +2026-05-05 03:52:10,879 INFO MainThread:3081796 [wandb_init.py:init():973] updated telemetry +2026-05-05 03:52:10,908 INFO MainThread:3081796 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 03:52:12,109 INFO MainThread:3081796 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 03:52:12,506 INFO MainThread:3081796 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 03:52:12,506 INFO MainThread:3081796 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 03:52:12,506 INFO MainThread:3081796 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 03:52:12,506 INFO MainThread:3081796 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 03:52:12,514 INFO MainThread:3081796 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 04:20:59,201 INFO wandb-AsyncioManager-main:3081796 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 04:20:59,201 INFO wandb-AsyncioManager-main:3081796 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/config.yaml b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69506988d0c42542930d2a12f222afa05ad8902c --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/config.yaml @@ -0,0 +1,403 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + 6ajwbambigvd9bcps8criiiye4tiw2k0: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_14b_lora_t769_critic_lr4e7.py + - '-' + - trainer.resume=False + - log_config.group=omniavatar_sf_audiofix + - log_config.name=sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - trainer.max_iter=2500 + - trainer.save_ckpt_iter=100 + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "1176008298496" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28 + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7 + startedAt: "2026-05-04T18:52:10.103903Z" + writerId: 6ajwbambigvd9bcps8criiiye4tiw2k0 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_audiofix + name: sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + lora_alpha: "64" + lora_rank: "128" + merge_lora: "False" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + unfreeze_modules: + - model.audio_proj + - model.audio_cond_projs + - model.patch_embedding + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "4e-07" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "True" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + lora_alpha: "64" + lora_rank: "128" + merge_lora: "False" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + unfreeze_modules: + - _core.audio_proj + - _core.audio_cond_projs + - _core.patch_embedding + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + reverse: "False" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_14b_lora_t769_4gpu_bs4_grad1_eff16_lr1e5_5000iter/checkpoints/0003000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "100000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "4" + logging_iter: "1" + max_iter: "2500" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/output.log b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2e128f5d4ebf59a054a7583ddd03e85c0fd01b7e --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/output.log @@ -0,0 +1,410 @@ +[May 5, 2026 - 03:52:12 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[May 5, 2026 - 03:52:12 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[May 5, 2026 - 03:52:12 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[May 5, 2026 - 03:52:12 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[May 5, 2026 - 03:52:12 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[May 5, 2026 - 03:52:12 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[May 5, 2026 - 03:52:12 | INFO | fastgen.trainer:run:77 ] Starting training +[May 5, 2026 - 03:52:12 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[May 5, 2026 - 03:52:12 | INFO | fastgen.utils.checkpointer:load:415 ] Loading FSDP model (.pth metadata stub at /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_14b_lora_t769_4gpu_bs4_grad1_eff16_lr1e5_5000iter/checkpoints/0003000.pth, sharded state in sibling *.{k}_model dirs) +[May 5, 2026 - 03:52:12 | INFO | fastgen.utils.checkpointer:load:436 ] Loading FSDP model from prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_14b_lora_t769_4gpu_bs4_grad1_eff16_lr1e5_5000iter/checkpoints/0003000 +[May 5, 2026 - 03:52:12 | INFO | fastgen.utils.checkpointer:load:439 ] Loading the FSDP model dict for key net... +[May 5, 2026 - 03:52:12 | INFO | fastgen.utils.checkpointer:load:495 ] [FSDPCheckpointer] net_model: loading 844/1937 state-dict entries (skipping 1093 frozen-param entries) +[May 5, 2026 - 03:52:52 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_14b_lora_t769_4gpu_bs4_grad1_eff16_lr1e5_5000iter/checkpoints/0003000.pth at iteration 3000 +[May 5, 2026 - 03:52:52 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 3000. +[May 5, 2026 - 03:52:52 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[May 5, 2026 - 03:53:32 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[May 5, 2026 - 03:53:32 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[May 5, 2026 - 03:53:32 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[May 5, 2026 - 03:53:32 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[May 5, 2026 - 03:53:32 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (14.91B params)... +[May 5, 2026 - 03:53:50 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:142 ] [Rank 0] Extracted state dict with 1937 tensors +[May 5, 2026 - 03:53:50 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2431 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[May 5, 2026 - 03:54:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 03:54:02 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:185 ] Syncing module states from rank 0 to all ranks +[May 5, 2026 - 03:54:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 40.9s +[May 5, 2026 - 03:54:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (14.91B params)... +[May 5, 2026 - 03:55:05 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:142 ] [Rank 0] Extracted state dict with 1937 tensors +[May 5, 2026 - 03:55:05 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 03:55:24 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 03:55:24 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:185 ] Syncing module states from rank 0 to all ranks +[May 5, 2026 - 03:55:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 93.4s +[May 5, 2026 - 03:55:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[May 5, 2026 - 03:55:56 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:142 ] [Rank 0] Extracted state dict with 1137 tensors +[May 5, 2026 - 03:55:56 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[May 5, 2026 - 03:56:09 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[May 5, 2026 - 03:56:09 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:185 ] Syncing module states from rank 0 to all ranks +[May 5, 2026 - 03:56:22 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 35.0s +[May 5, 2026 - 03:56:22 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[May 5, 2026 - 03:56:22 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[May 5, 2026 - 03:56:22 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1272 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 1.38M params in '_core.audio_proj' +[May 5, 2026 - 03:56:22 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1272 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 3.21M params in '_core.audio_cond_projs' +[May 5, 2026 - 03:56:22 | INFO | fastgen.networks.OmniAvatar.network_causal:_apply_unfreeze:1272 ] [CausalOmniAvatarWan] unfreeze: re-enabled requires_grad on 1.34M params in '_core.patch_embedding' +[May 5, 2026 - 03:56:22 | INFO | fastgen.networks.OmniAvatar.network_causal:apply_lora_freeze:1234 ] [CausalOmniAvatarWan] apply_lora_freeze: LoRA 613.42M trainable, base 14294.08M frozen, then unfreeze_modules re-enabled on top +[May 5, 2026 - 03:56:22 | INFO | fastgen.networks.OmniAvatar.network:_apply_unfreeze:691 ] [OmniAvatarWan] unfreeze: re-enabled requires_grad on 1.38M params in 'model.audio_proj' +[May 5, 2026 - 03:56:22 | INFO | fastgen.networks.OmniAvatar.network:_apply_unfreeze:691 ] [OmniAvatarWan] unfreeze: re-enabled requires_grad on 3.21M params in 'model.audio_cond_projs' +[May 5, 2026 - 03:56:22 | INFO | fastgen.networks.OmniAvatar.network:_apply_unfreeze:691 ] [OmniAvatarWan] unfreeze: re-enabled requires_grad on 1.34M params in 'model.patch_embedding' +[May 5, 2026 - 03:56:22 | INFO | fastgen.networks.OmniAvatar.network:apply_lora_freeze:747 ] [OmniAvatarWan] apply_lora_freeze: LoRA 613.42M trainable, base 14294.08M frozen, then unfreeze_modules re-enabled on top +[May 5, 2026 - 03:56:22 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[May 5, 2026 - 03:56:22 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[May 5, 2026 - 03:56:22 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 1238.68 M trainable and 44122.71 M total params (logical). +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 309.67 M trainable and 11043.69 M total params LOCAL on rank 0. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 309.67 M trainable and 11039.97 M total params LOCAL on rank 1. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 309.67 M trainable and 11039.97 M total params LOCAL on rank 2. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 309.67 M trainable and 11039.97 M total params LOCAL on rank 3. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 619.34 M trainable and 14907.50 M total params (logical). +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 154.83 M trainable and 3727.80 M total params LOCAL on rank 0. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 154.83 M trainable and 3726.56 M total params LOCAL on rank 1. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 154.83 M trainable and 3726.56 M total params LOCAL on rank 2. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 154.83 M trainable and 3726.56 M total params LOCAL on rank 3. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 619.34 M trainable and 14907.50 M total params (logical). +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 154.83 M trainable and 3727.80 M total params LOCAL on rank 0. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 154.83 M trainable and 3726.56 M total params LOCAL on rank 1. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 154.83 M trainable and 3726.56 M total params LOCAL on rank 2. +[May 5, 2026 - 03:56:28 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 154.83 M trainable and 3726.56 M total params LOCAL on rank 3. +[May 5, 2026 - 03:56:28 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.124518 34.182526 4.768955 +peak_gpu_mem_gb 87.498918 87.498918 87.498918 +peak_gpu_mem_reserved_gb 90.663086 90.677734 90.658203 +util 78.250000 86.000000 69.000000 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 0.59 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 9.73 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 6.71 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 0.88 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0051 iter count: 1.0 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0051 iter count: 1.0 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 03:57:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.134835 34.184456 4.781765 +peak_gpu_mem_gb 88.653263 88.653263 88.653263 +peak_gpu_mem_reserved_gb 90.949707 90.964844 90.904297 +util 92.000000 99.000000 87.000000 +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 63.01 seconds +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 8.89 +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 6.71 +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 0.68 +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0051 iter count: 1.0 +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0051 iter count: 1.0 +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.136192 34.183857 4.783360 +peak_gpu_mem_gb 88.652531 88.652531 88.652531 +peak_gpu_mem_reserved_gb 90.918945 90.923828 90.904297 +util 96.750000 99.000000 90.000000 +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 65.16 seconds +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 9.43 +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 6.70 +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 0.68 +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0032 iter count: 1.0 +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0032 iter count: 1.0 +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 03:59:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.136497 34.184811 4.783447 +peak_gpu_mem_gb 88.652531 88.652531 88.652531 +peak_gpu_mem_reserved_gb 90.918945 90.923828 90.904297 +util 95.750000 99.000000 93.000000 +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 60.92 seconds +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 8.37 +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 6.70 +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 0.68 +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0048 iter count: 1.0 +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0048 iter count: 1.0 +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:00:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.612702 34.259380 5.393940 +peak_gpu_mem_gb 108.783676 108.783676 108.783676 +peak_gpu_mem_reserved_gb 137.165527 137.375000 136.537109 +util 39.000000 43.000000 31.000000 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 154.27 seconds +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 31.15 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 7.63 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.65 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 7658.6073 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4918 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 7658.6073iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3658iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 3.1757 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 5.4854 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 20030.8398iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 573.3256iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 58147.0742iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4918iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.4854 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 58147.0742iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.8736 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2315.0237iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1757 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 573.3256iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9284 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 19087.9316iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0045 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:03:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.613334 34.260799 5.393982 +peak_gpu_mem_gb 90.250821 90.250821 90.250821 +peak_gpu_mem_reserved_gb 92.577637 92.578125 92.576172 +util 96.500000 99.000000 94.000000 +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 65.55 seconds +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 9.52 +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 6.72 +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 0.68 +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:04:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.613925 34.260799 5.396294 +peak_gpu_mem_gb 90.250089 90.250089 90.250089 +peak_gpu_mem_reserved_gb 92.592773 92.597656 92.578125 +util 90.250000 95.000000 87.000000 +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 63.12 seconds +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 8.91 +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 6.71 +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 0.69 +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0040 iter count: 1.0 +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0040 iter count: 1.0 +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:05:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.614724 34.260799 5.397278 +peak_gpu_mem_gb 90.250089 90.250089 90.250089 +peak_gpu_mem_reserved_gb 92.592773 92.597656 92.578125 +util 96.500000 99.000000 89.000000 +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 63.12 seconds +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 8.92 +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 6.69 +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 0.69 +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 8-------------------- +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0047 iter count: 1.0 +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0047 iter count: 1.0 +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:06:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.614725 34.260818 5.397263 +peak_gpu_mem_gb 90.250821 90.250821 90.250821 +peak_gpu_mem_reserved_gb 92.578125 92.578125 92.578125 +util 93.500000 98.000000 90.000000 +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 67.11 seconds +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 9.91 +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 6.71 +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 0.68 +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 9-------------------- +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0044 iter count: 1.0 +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0044 iter count: 1.0 +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:07:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.620036 34.260208 5.402706 +peak_gpu_mem_gb 109.937289 109.937289 109.937289 +peak_gpu_mem_reserved_gb 137.143555 137.382812 136.425781 +util 63.500000 68.000000 61.000000 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 158.83 seconds +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 32.00 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 7.69 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.40 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 10-------------------- +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 37204.8936 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.3624 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 37204.8936iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0558iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 4.0237 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 5.8440 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 55807.0117iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3125.8291iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 119138.2422iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.3624iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0237 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3125.8291iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.7037 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 89992.5469iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6515 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 10971.4424iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.8440 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 119138.2422iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0033 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:10:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.621252 34.260216 5.406338 +peak_gpu_mem_gb 90.250089 90.250089 90.250089 +peak_gpu_mem_reserved_gb 92.572754 92.615234 92.558594 +util 97.750000 99.000000 96.000000 +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 67.43 seconds +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 9.99 +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 6.70 +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 0.68 +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 11-------------------- +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0054 iter count: 1.0 +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0054 iter count: 1.0 +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:11:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.621604 34.261505 5.406353 +peak_gpu_mem_gb 90.250089 90.250089 90.250089 +peak_gpu_mem_reserved_gb 92.573242 92.617188 92.558594 +util 93.250000 98.000000 90.000000 +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 63.33 seconds +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 8.96 +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 6.71 +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 0.69 +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 12-------------------- +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0058 iter count: 1.0 +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:12:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.621794 34.262260 5.406353 +peak_gpu_mem_gb 90.250089 90.250089 90.250089 +peak_gpu_mem_reserved_gb 92.573242 92.617188 92.558594 +util 91.750000 99.000000 87.000000 +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 64.94 seconds +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 9.37 +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 6.71 +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 0.68 +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 13-------------------- +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0054 iter count: 1.0 +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0054 iter count: 1.0 +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:13:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.621888 34.262634 5.406368 +peak_gpu_mem_gb 90.250821 90.250821 90.250821 +peak_gpu_mem_reserved_gb 92.573242 92.617188 92.558594 +util 91.500000 96.000000 88.000000 +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 65.24 seconds +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 9.45 +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 6.72 +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 0.68 +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 14-------------------- +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0040 iter count: 1.0 +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0040 iter count: 1.0 +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:14:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.622731 34.262371 5.408104 +peak_gpu_mem_gb 109.937289 109.937289 109.937289 +peak_gpu_mem_reserved_gb 136.899414 137.070312 136.386719 +util 65.500000 70.000000 62.000000 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 155.65 seconds +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 31.33 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 7.62 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.38 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 15-------------------- +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 11538.2202 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.0087 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 11538.2202iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.7298iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 3.1835 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3877 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1231164.1250iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 582.3253iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2611418.0000iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.0087iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.0261 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 23207.5898iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.1835 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 582.3253iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.3877 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2611418.0000iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.3219 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2289449.0000iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0054 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:17:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.622730 34.262371 5.408096 +peak_gpu_mem_gb 90.250089 90.250089 90.250089 +peak_gpu_mem_reserved_gb 92.563477 92.578125 92.558594 +util 95.250000 99.000000 92.000000 +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 65.59 seconds +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 9.53 +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 6.70 +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 0.68 +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 16-------------------- +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0052 iter count: 1.0 +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0052 iter count: 1.0 +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:18:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.622730 34.262371 5.408096 +peak_gpu_mem_gb 90.250089 90.250089 90.250089 +peak_gpu_mem_reserved_gb 92.563477 92.578125 92.558594 +util 95.000000 99.000000 92.000000 +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 60.99 seconds +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 8.38 +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 6.71 +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 0.68 +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 17-------------------- +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0034 iter count: 1.0 +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0034 iter count: 1.0 +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:19:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 12.622985 34.262371 5.408176 +peak_gpu_mem_gb 90.250089 90.250089 90.250089 +peak_gpu_mem_reserved_gb 92.534180 92.578125 92.519531 +util 93.750000 99.000000 86.000000 +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 63.10 seconds +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 8.91 +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 6.72 +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 0.67 +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 18-------------------- +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0049 iter count: 1.0 +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0049 iter count: 1.0 +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 5, 2026 - 04:20:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/requirements.txt b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/wandb-metadata.json b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b2ba43536a334b43fccd459a7e4366844c9f8e06 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-05-04T18:52:10.103903Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_14b_lora_t769_critic_lr4e7.py", + "-", + "trainer.resume=False", + "log_config.group=omniavatar_sf_audiofix", + "log_config.name=sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "trainer.max_iter=2500", + "trainer.save_ckpt_iter=100" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "de6ba8eeeb976c1a68b7151a9ed4e986db2f8e28" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1176008298496" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "6ajwbambigvd9bcps8criiiye4tiw2k0" +} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/wandb-summary.json b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3e4b1f3abf04752822ea9a2283f3d1046ad3e330 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/gan_loss_disc":0,"net/local_trainable_params":[154834968,154834968,154834968,154834968],"optimizer/lr_fake_score":4e-07,"train/reward_sync_c_r1":3.1835145950317383,"train/reward_weight_mean":1.231164125e+06,"net/total_params":14907498656,"profiler/avg_iteration_time":63.09861660003662,"train/vsd_loss":0.0086669921875,"train/reward_sync_c_mean":5.729811191558838,"model/local_total_params":[11043689208,11039972088,11039972088,11039972088],"_step":18,"train/reward_sync_c_r2":7.387701988220215,"train/reward_weight_r2":2.611418e+06,"model/local_trainable_params":[309669936,309669936,309669936,309669936],"net/trainable_params":619339872,"train/vsd_loss_unweighted":0.0086669921875,"train/reward_weight_min":582.3252563476562,"fake_score/local_trainable_params":[154834968,154834968,154834968,154834968],"optimizer/iteration":18,"train/reward_weight_max":2.611418e+06,"train/reward_sync_c_r3":7.321910858154297,"train/vsd_loss_weighted":11538.220197439194,"profiler/backward_pass_time":6.71941022016108,"train/reward_weight_r1":582.3252563476562,"model/trainable_params":1238679744,"_timestamp":1.7779224240583296e+09,"train/reward_weight_r3":2.289449e+06,"train/fake_score_loss":0.00493621826171875,"_wandb":{"runtime":1727},"net/local_total_params":[3727803944,3726564904,3726564904,3726564904],"train/total_loss":0.00493621826171875,"profiler/data_loading_time":0.000680943951010704,"train/gan_loss_gen":0,"model/total_params":44122710624,"train/reward_sync_c_min":3.1835145950317383,"fake_score/local_total_params":[3727803944,3726564904,3726564904,3726564904],"profiler/avg_forward_pass_time":8.907583605963737,"train/reward_sync_c_r0":5.026117324829102,"fake_score/total_params":14907498656,"train/reward_weight_r0":23207.58984375,"optimizer/grad_norm (model_key net)":0,"optimizer/lr_net":2e-06,"_runtime":1727.08974144,"train/reward_sync_c_max":7.387701988220215,"optimizer/non_finite_grads_count (model_key net)":0,"profiler/optimizer_step_time":0.6725488658994436,"fake_score/trainable_params":619339872} \ No newline at end of file diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug-core.log b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f24ff0548604cb0d0e3a6e07e4d3456b3e0e7350 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2026-05-05T03:52:10.763506386+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpxk6btlc1/port-3081796.txt","pid":3081796,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-05T03:52:10.766385436+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3081796-3143282-4051710815/socket","Net":"unix"}} +{"time":"2026-05-05T03:52:10.76695873+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3081796} +{"time":"2026-05-05T03:52:10.853604218+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-05T03:52:10.882484574+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"fxrufsq8","id":"1(@)"} +{"time":"2026-05-05T03:52:11.41790318+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fxrufsq8","id":"1(@)"} +{"time":"2026-05-05T03:52:17.510289684+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"bqivn3erf54j"} +{"time":"2026-05-05T04:20:59.201542394+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-05T04:20:59.201598923+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-05T04:20:59.201585816+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-05T04:20:59.202450542+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-05T04:20:59.202454067+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3081796-3143282-4051710815/socket","Net":"unix"}} diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug-internal.log b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2061d7dbe2c6f030cc8a08711de12a118a77ff30 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-05T03:52:10.883134411+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-05-05T03:52:11.417276647+09:00","level":"INFO","msg":"stream: created new stream","id":"fxrufsq8"} +{"time":"2026-05-05T03:52:11.417813903+09:00","level":"INFO","msg":"handler: started","stream_id":"fxrufsq8"} +{"time":"2026-05-05T03:52:11.417896447+09:00","level":"INFO","msg":"stream: started","id":"fxrufsq8"} +{"time":"2026-05-05T03:52:11.417921032+09:00","level":"INFO","msg":"sender: started","stream_id":"fxrufsq8"} +{"time":"2026-05-05T03:52:11.417945996+09:00","level":"INFO","msg":"writer: started","stream_id":"fxrufsq8"} +{"time":"2026-05-05T04:20:59.201594537+09:00","level":"INFO","msg":"stream: closing","id":"fxrufsq8"} +{"time":"2026-05-05T04:20:59.821709487+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-05T04:21:00.158020868+09:00","level":"INFO","msg":"handler: closed","stream_id":"fxrufsq8"} +{"time":"2026-05-05T04:21:00.158092039+09:00","level":"INFO","msg":"sender: closed","stream_id":"fxrufsq8"} +{"time":"2026-05-05T04:21:00.158108163+09:00","level":"INFO","msg":"stream: closed","id":"fxrufsq8"} diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug.log b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b8ce0c7c15cec8690e89a35385d199e0c50cf9db --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_setup.py:_flush():81] Configure stats pid to 3081796 +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug.log +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/logs/debug-internal.log +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:init():844] calling init triggers +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_14b_lora_t769_4gpu_bs4_grad1_eff16_lr1e5_5000iter/checkpoints/0003000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '2500', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '4', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '100000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': "", 'merge_lora': 'False', 'unfreeze_modules': ['_core.audio_proj', '_core.audio_cond_projs', '_core.patch_embedding'], 'lora_rank': '128', 'lora_alpha': '64'}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': "", 'unfreeze_modules': ['model.audio_proj', 'model.audio_cond_projs', 'model.patch_embedding'], 'lora_rank': '128', 'lora_alpha': '64'}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882', 'reverse': 'False'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'True', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '4e-07', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-05-05 03:52:10,105 INFO MainThread:3081796 [wandb_init.py:init():892] starting backend +2026-05-05 03:52:10,853 INFO MainThread:3081796 [wandb_init.py:init():895] sending inform_init request +2026-05-05 03:52:10,878 INFO MainThread:3081796 [wandb_init.py:init():903] backend started and connected +2026-05-05 03:52:10,879 INFO MainThread:3081796 [wandb_init.py:init():973] updated telemetry +2026-05-05 03:52:10,908 INFO MainThread:3081796 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-05 03:52:12,109 INFO MainThread:3081796 [wandb_init.py:init():1042] starting run threads in backend +2026-05-05 03:52:12,506 INFO MainThread:3081796 [wandb_run.py:_console_start():2524] atexit reg +2026-05-05 03:52:12,506 INFO MainThread:3081796 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-05-05 03:52:12,506 INFO MainThread:3081796 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-05-05 03:52:12,506 INFO MainThread:3081796 [wandb_run.py:_redirect():2465] Redirects installed. +2026-05-05 03:52:12,514 INFO MainThread:3081796 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-05 04:20:59,201 INFO wandb-AsyncioManager-main:3081796 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-05 04:20:59,201 INFO wandb-AsyncioManager-main:3081796 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/run-fxrufsq8.wandb b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/run-fxrufsq8.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7560b192a69ee450db588d463c5c94ae7ff7b79b --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb/run-20260505_035210-fxrufsq8/run-fxrufsq8.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb131a2259cb327b3cc76ebf3440cb5c3bda0880d463ec1f409715590bd92e9b +size 376781 diff --git a/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb_id.txt b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..7ee8baf1e6500109f83ba7c7f96527ba08709f85 --- /dev/null +++ b/FastGen-redmd_outputs/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769_14B_LORA_CRITIC_LR4E7/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769_14b_lora_critic_lr4e7/wandb_id.txt @@ -0,0 +1 @@ +fxrufsq8